From 15fa0bf0eb64b7f02fd9c6e9532d1a4a79ccf0ab Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 13 Feb 2024 12:59:03 +0100 Subject: [PATCH 01/78] Avoid false-positive matches on intermediate objects in `ecs@mappings` (#105440) Fixes #102794 --- docs/changelog/105440.yaml | 6 +++ .../src/main/resources/ecs@mappings.json | 46 +++++++------------ .../xpack/stack/EcsDynamicTemplatesIT.java | 28 +++++++++++ .../xpack/stack/StackTemplateRegistry.java | 2 +- 4 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 docs/changelog/105440.yaml diff --git a/docs/changelog/105440.yaml b/docs/changelog/105440.yaml new file mode 100644 index 0000000000000..8aacac3e641bf --- /dev/null +++ b/docs/changelog/105440.yaml @@ -0,0 +1,6 @@ +pr: 105440 +summary: Avoid false-positive matches on intermediate objects in `ecs@mappings` +area: Data streams +type: bug +issues: + - 102794 diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json index a91cf5d1606c6..7eaf37ba1d95e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json @@ -19,7 +19,8 @@ "path_match": [ "message", "*.message" - ] + ], + "unmatch_mapping_type": "object" } }, { @@ -45,7 +46,8 @@ "*.message_id", "*registry.data.strings", "*url.path" - ] + ], + "unmatch_mapping_type": "object" } }, { @@ -62,7 +64,8 @@ "*.body.content", "*url.full", "*url.original" - ] + ], + "unmatch_mapping_type": "object" } }, { @@ -78,7 +81,8 @@ "match": [ "*command_line", "*stack_trace" - ] + ], + "unmatch_mapping_type": "object" } }, { @@ -103,7 +107,8 @@ "email.subject", "vulnerability.description", "user_agent.original" - ] + ], + "unmatch_mapping_type": "object" } }, { @@ -127,7 +132,8 @@ "*.ingested", "*.start", "*.end" - ] + ], + "unmatch_mapping_type": "object" } }, { @@ -139,7 +145,8 @@ "*.score.*", "*_score*" ], - "path_unmatch": "*.version" + "path_unmatch": "*.version", + "unmatch_mapping_type": "object" } }, { @@ -149,27 +156,7 @@ "scaling_factor": 1000 }, "path_match": "*.usage", - "match_mapping_type": "double" - } - }, - { - "ecs_usage_long_scaled_float": { - "mapping": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "path_match": "*.usage", - "match_mapping_type": "long" - } - }, - { - "ecs_usage_long_scaled_float": { - "mapping": { - "type": "scaled_float", - "scaling_factor": 1000 - }, - "path_match": "*.usage", - "match_mapping_type": "string" + "match_mapping_type": ["double", "long", "string"] } }, { @@ -192,7 +179,8 @@ "*structured_data", "*exports", "*imports" - ] + ], + "match_mapping_type": "object" } }, { diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index 75588baf6e6f4..09e9a6090c485 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -214,6 +214,34 @@ public void testUsage() throws IOException { assertEquals("float", flatFieldMappings.get("root.usage.float")); } + public void testOnlyMatchLeafFields() throws IOException { + // tests that some of the match conditions only apply to leaf fields, not intermediate objects + String indexName = "test"; + createTestIndex(indexName); + Map fieldsMap = createTestDocument(false); + fieldsMap.put("foo.message.bar", 123); + fieldsMap.put("foo.url.path.bar", 123); + fieldsMap.put("foo.url.full.bar", 123); + fieldsMap.put("foo.stack_trace.bar", 123); + fieldsMap.put("foo.user_agent.original.bar", 123); + fieldsMap.put("foo.created.bar", 123); + fieldsMap.put("foo._score.bar", 123); + fieldsMap.put("foo.structured_data", 123); + indexDocument(indexName, fieldsMap); + + final Map rawMappings = getMappings(indexName); + final Map flatFieldMappings = new HashMap<>(); + processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); + assertEquals("long", flatFieldMappings.get("foo.message.bar")); + assertEquals("long", flatFieldMappings.get("foo.url.path.bar")); + assertEquals("long", flatFieldMappings.get("foo.url.full.bar")); + assertEquals("long", flatFieldMappings.get("foo.stack_trace.bar")); + assertEquals("long", flatFieldMappings.get("foo.user_agent.original.bar")); + assertEquals("long", flatFieldMappings.get("foo.created.bar")); + assertEquals("float", flatFieldMappings.get("foo._score.bar")); + assertEquals("long", flatFieldMappings.get("foo.structured_data")); + } + private static void indexDocument(String indexName, Map flattenedFieldsMap) throws IOException { try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { Request indexRequest = new Request("POST", "/" + indexName + "/_doc"); diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index c2202acfc7815..1eaf224083c87 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -43,7 +43,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { // The stack template registry version. This number must be incremented when we make changes // to built-in templates. - public static final int REGISTRY_VERSION = 7; + public static final int REGISTRY_VERSION = 8; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting( From 1f0ea3e015a789bdb71dc4c65b7bbf8f1bdf7d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Slobodan=20Adamovi=C4=87?= Date: Tue, 13 Feb 2024 14:04:32 +0100 Subject: [PATCH 02/78] [Doc] Mark `secure_bind_password` settings as reloadable (#105448) This is a followup to #104320, which updates the docs for `secure_bind_password` settings and marks them as reloadable. --- docs/reference/settings/security-settings.asciidoc | 4 ++-- docs/reference/setup/secure-settings.asciidoc | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc index 6a60a8a6703fe..fee38383896fa 100644 --- a/docs/reference/settings/security-settings.asciidoc +++ b/docs/reference/settings/security-settings.asciidoc @@ -404,7 +404,7 @@ Defaults to Empty. Due to its potential security impact, `bind_password` is not exposed via the <>. `secure_bind_password`:: -(<>) +(<>, <>) The password for the user that is used to bind to the LDAP directory. Defaults to Empty. @@ -749,7 +749,7 @@ potential security impact, `bind_password` is not exposed via the <>. `secure_bind_password`:: -(<>) +(<>, <>) The password for the user that is used to bind to Active Directory. Defaults to Empty. diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc index 22e828f96f5d2..028e766d063df 100644 --- a/docs/reference/setup/secure-settings.asciidoc +++ b/docs/reference/setup/secure-settings.asciidoc @@ -61,3 +61,6 @@ There are reloadable secure settings for: * <> * <> * <> +* <> +* <> +* <> From e1217e41db260ddef890f27a752343b890c7deb2 Mon Sep 17 00:00:00 2001 From: Nassim Kammah Date: Tue, 13 Feb 2024 14:48:02 +0100 Subject: [PATCH 03/78] Doc update (#105270) Following the migration from Jenkins to Buildkite, docs previews are now available at _bk_. More context in https://github.com/elastic/docs/pull/2898 --- docs/README.asciidoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/README.asciidoc b/docs/README.asciidoc index c4f956833aa73..9b7e280e532f5 100644 --- a/docs/README.asciidoc +++ b/docs/README.asciidoc @@ -19,17 +19,17 @@ https://github.com/elastic/docs#readme[elastic/docs README]. === Preview links for PRs -For open PRs, the `elasticsearch-ci/docs` check generates a live preview of any +For open PRs, a Buildkite job called `docs-build-pr` generates a live preview of any docs changes. If the check runs successfully, you can find the preview at: -`https://elasticsearch_.docs-preview.app.elstc.co/diff` +`https://elasticsearch_bk_.docs-preview.app.elstc.co/diff` -`elasticsearch-ci/docs` runs automatically for PRs opened by Elastic employees. +`docs-build-pr` runs automatically for PRs opened by Elastic employees. To run CI checks on PRs from external contributors, an Elastic employee can -leave a GitHub comment containing `@elasticmachine ok to test`. +leave a GitHub comment containing `run docs-build`. -To re-run `elasticsearch-ci/docs`, an Elastic employee can leave a GitHub -comment containing `@elasticmachine run elasticsearch-ci/docs`. +To re-run `docs-build-pr`, an Elastic employee can leave a GitHub +comment containing `run docs-build`. == Add API reference pages From 77a5d15cc7c4f81afa9c36b8bf035583efa64225 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Tue, 13 Feb 2024 16:12:17 +0200 Subject: [PATCH 04/78] Wait for all cluster state updates to be processed before wiping the cluster (#104846) (#105349) In #104846, we see an inconsistency in the state of the clusters between the test and the clean up at the end of the test. The failure is as follows: - The test using ILM at some point creates a full searchable snapshots with name `restore-my-index-xxxx` - Then ILM moves to frozen and creates a partially searchable snapshots with alias `restore-my-index-xxxx` - The test confirms that `restore-my-index-xxxx` is an alias and ends - During tear down, it appears that the cluster state retrieved contains the `restore-my-index-xxxx` as an index so it issues a request to delete it. - The deletion fails because `restore-my-index-xxxx` is an alias. I do not think that the test has an issue, most of the clues shows that the partial searchable snapshot has been correctly processed. Only this cluster state retrieval seems a bit off. In order to reduce this flakiness we introduce a `GET _cluster/health?wait_for_events=languid` to ensure we get the latest cluster state. Fixes #104846 --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 1860283515c9d..6520e3d0f68bd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -813,6 +813,8 @@ protected boolean preserveSearchableSnapshotsIndicesUponCompletion() { } private void wipeCluster() throws Exception { + logger.info("Waiting for all cluster updates up to this moment to be processed"); + assertOK(adminClient().performRequest(new Request("GET", "_cluster/health?wait_for_events=languid"))); // Cleanup rollup before deleting indices. A rollup job might have bulks in-flight, // so we need to fully shut them down first otherwise a job might stall waiting From 87c23c734ceee039abad1d7a374b80f935ebc17d Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 13 Feb 2024 15:14:29 +0000 Subject: [PATCH 05/78] [DSL] Avoid reading the PREFER_ILM setting until needed (#105446) --- .../java/org/elasticsearch/cluster/metadata/DataStream.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 8cb4b39efac78..14de79636be0d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -814,10 +814,9 @@ public boolean isIndexManagedByDataStreamLifecycle(Index index, Function Date: Tue, 13 Feb 2024 08:15:59 -0800 Subject: [PATCH 06/78] Update Gradle Enterprise plugin to 3.16.1 (#104435) --- gradle/build.versions.toml | 2 +- gradle/verification-metadata.xml | 6 +++--- plugins/examples/settings.gradle | 2 +- settings.gradle | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index 6960ab39b48cf..a17f77c6b4917 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -17,7 +17,7 @@ commons-codec = "commons-codec:commons-codec:1.11" commmons-io = "commons-io:commons-io:2.2" docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5" forbiddenApis = "de.thetaphi:forbiddenapis:3.6" -gradle-enterprise = "com.gradle:gradle-enterprise-gradle-plugin:3.14.1" +gradle-enterprise = "com.gradle:gradle-enterprise-gradle-plugin:3.16.1" hamcrest = "org.hamcrest:hamcrest:2.1" httpcore = "org.apache.httpcomponents:httpcore:4.4.12" httpclient = "org.apache.httpcomponents:httpclient:4.5.14" diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index eb1df32798e58..ae2ec2345d185 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -736,9 +736,9 @@ - - - + + + diff --git a/plugins/examples/settings.gradle b/plugins/examples/settings.gradle index 3b6280dc3bcbd..f71eca0d1f966 100644 --- a/plugins/examples/settings.gradle +++ b/plugins/examples/settings.gradle @@ -7,7 +7,7 @@ */ plugins { - id "com.gradle.enterprise" version "3.14.1" + id "com.gradle.enterprise" version "3.16.1" } // Include all subdirectories as example projects diff --git a/settings.gradle b/settings.gradle index 8cff387d3d925..5eefe21b360d6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,7 +14,7 @@ pluginManagement { } plugins { - id "com.gradle.enterprise" version "3.14.1" + id "com.gradle.enterprise" version "3.16.1" id 'elasticsearch.java-toolchain' } From b5828fbb671cae81a9f9818fe3370a00628a1a62 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 13 Feb 2024 08:30:04 -0800 Subject: [PATCH 07/78] Add plumbing to check cluster features in SearchSourceBuilder (#105417) This change adds additional plumbing to pipe through the available cluster features into SearchSourceBuilder. A number of different APIs use SearchSourceBuilder so they had to make this available through their parsers as well often through ParserContext. This change is largely mechanical passing a Predicate into existing REST actions to check for feature availability. Note that this change was pulled mostly from this PR (#105040). --- .../script/mustache/MustachePlugin.java | 2 +- .../mustache/RestSearchTemplateAction.java | 7 ++- .../TransportMultiSearchTemplateAction.java | 23 ++++++- .../TransportSearchTemplateAction.java | 26 +++++++- .../RestSearchTemplateActionTests.java | 2 +- .../index/rankeval/RankEvalPlugin.java | 2 +- .../index/rankeval/RankEvalSpec.java | 10 ++-- .../index/rankeval/RatedRequest.java | 10 ++-- .../index/rankeval/RestRankEvalAction.java | 19 +++++- .../rankeval/TransportRankEvalAction.java | 16 ++++- .../index/rankeval/RankEvalSpecTests.java | 4 +- .../index/rankeval/RatedRequestsTests.java | 6 +- .../TransportRankEvalActionTests.java | 6 +- .../AbstractBulkByQueryRestHandler.java | 12 +++- .../elasticsearch/reindex/ReindexPlugin.java | 6 +- .../reindex/RestDeleteByQueryAction.java | 8 ++- .../reindex/RestReindexAction.java | 8 ++- .../reindex/RestUpdateByQueryAction.java | 8 ++- .../reindex/RestDeleteByQueryActionTests.java | 2 +- .../reindex/RestReindexActionTests.java | 2 +- .../reindex/RestUpdateByQueryActionTests.java | 2 +- .../bucket/terms/StringTermsIT.java | 17 +++--- .../elasticsearch/action/ActionModule.java | 6 +- .../index/reindex/ReindexRequest.java | 18 ++++-- .../action/search/RestMultiSearchAction.java | 34 +++++++++-- .../rest/action/search/RestSearchAction.java | 30 ++++++++-- .../search/builder/SearchSourceBuilder.java | 38 ++++++++---- .../search/vectors/KnnSearchBuilder.java | 2 +- .../search/MultiSearchRequestTests.java | 30 +++++++--- .../index/reindex/ReindexRequestTests.java | 6 +- .../search/RestMultiSearchActionTests.java | 7 ++- .../action/search/RestSearchActionTests.java | 2 +- .../builder/SearchSourceBuilderTests.java | 60 ++++++++++--------- .../xpack/search/AsyncSearch.java | 2 +- .../search/RestSubmitAsyncSearchAction.java | 20 ++++++- .../RestSubmitAsyncSearchActionTests.java | 6 +- .../SearchApplicationTemplateService.java | 12 +++- ...TransportQuerySearchApplicationAction.java | 14 ++++- ...ortRenderSearchApplicationQueryAction.java | 13 +++- .../org/elasticsearch/xpack/fleet/Fleet.java | 4 +- .../rest/RestFleetMultiSearchAction.java | 8 ++- .../fleet/rest/RestFleetSearchAction.java | 20 ++++++- .../qa/native-multi-node-tests/build.gradle | 1 + .../xpack/rank/rrf/RRFRankPlugin.java | 3 +- .../test/license/100_license.yml | 1 - .../rest-api-spec/test/rrf/100_rank_rrf.yml | 15 +++-- .../elasticsearch/xpack/rollup/Rollup.java | 2 +- .../rollup/rest/RestRollupSearchAction.java | 7 ++- .../elasticsearch/xpack/watcher/Watcher.java | 20 ++++++- .../input/search/SearchInputFactory.java | 12 +++- .../search/WatcherSearchTemplateService.java | 12 +++- .../search/SearchTransformFactory.java | 12 +++- .../test/integration/SearchInputTests.java | 6 +- .../integration/SearchTransformTests.java | 8 ++- .../xpack/watcher/watch/WatchTests.java | 10 +++- 55 files changed, 484 insertions(+), 155 deletions(-) diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index ebaa2f356733e..c698a603055ad 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -67,7 +67,7 @@ public List getRestHandlers( Predicate clusterSupportsFeature ) { return Arrays.asList( - new RestSearchTemplateAction(namedWriteableRegistry), + new RestSearchTemplateAction(namedWriteableRegistry, clusterSupportsFeature), new RestMultiSearchTemplateAction(settings), new RestRenderSearchTemplateAction() ); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index cfd726fd96fc3..a29c10b7501f1 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -23,6 +24,7 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -35,9 +37,11 @@ public class RestSearchTemplateAction extends BaseRestHandler { private static final Set RESPONSE_PARAMS = Set.of(TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM); private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestSearchTemplateAction(NamedWriteableRegistry namedWriteableRegistry) { + public RestSearchTemplateAction(NamedWriteableRegistry namedWriteableRegistry, Predicate clusterSupportsFeature) { this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -70,6 +74,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client request, null, namedWriteableRegistry, + clusterSupportsFeature, size -> searchRequest.source().size(size) ); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java index 11871978e433a..ba1d4dbaba012 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java @@ -18,8 +18,12 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; @@ -29,6 +33,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; import static org.elasticsearch.script.mustache.TransportSearchTemplateAction.convert; @@ -38,6 +43,7 @@ public class TransportMultiSearchTemplateAction extends HandledTransportAction clusterSupportsFeature; private final NodeClient client; private final SearchUsageHolder searchUsageHolder; @@ -48,7 +54,9 @@ public TransportMultiSearchTemplateAction( ScriptService scriptService, NamedXContentRegistry xContentRegistry, NodeClient client, - UsageService usageService + UsageService usageService, + ClusterService clusterService, + FeatureService featureService ) { super( MustachePlugin.MULTI_SEARCH_TEMPLATE_ACTION.name(), @@ -59,6 +67,10 @@ public TransportMultiSearchTemplateAction( ); this.scriptService = scriptService; this.xContentRegistry = xContentRegistry; + this.clusterSupportsFeature = f -> { + ClusterState state = clusterService.state(); + return state.clusterRecovered() && featureService.clusterHasFeature(state, f); + }; this.client = client; this.searchUsageHolder = usageService.getSearchUsageHolder(); } @@ -78,7 +90,14 @@ protected void doExecute(Task task, MultiSearchTemplateRequest request, ActionLi SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse(); SearchRequest searchRequest; try { - searchRequest = convert(searchTemplateRequest, searchTemplateResponse, scriptService, xContentRegistry, searchUsageHolder); + searchRequest = convert( + searchTemplateRequest, + searchTemplateResponse, + scriptService, + xContentRegistry, + clusterSupportsFeature, + searchUsageHolder + ); } catch (Exception e) { searchTemplateResponse.decRef(); items[i] = new MultiSearchTemplateResponse.Item(null, e); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index c6bd2afc64d21..74e4705447b64 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -13,10 +13,14 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; @@ -36,6 +40,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.function.Predicate; public class TransportSearchTemplateAction extends HandledTransportAction { @@ -43,6 +48,7 @@ public class TransportSearchTemplateAction extends HandledTransportAction clusterSupportsFeature; private final NodeClient client; private final SearchUsageHolder searchUsageHolder; @@ -53,7 +59,9 @@ public TransportSearchTemplateAction( ScriptService scriptService, NamedXContentRegistry xContentRegistry, NodeClient client, - UsageService usageService + UsageService usageService, + ClusterService clusterService, + FeatureService featureService ) { super( MustachePlugin.SEARCH_TEMPLATE_ACTION.name(), @@ -64,6 +72,10 @@ public TransportSearchTemplateAction( ); this.scriptService = scriptService; this.xContentRegistry = xContentRegistry; + this.clusterSupportsFeature = f -> { + ClusterState state = clusterService.state(); + return state.clusterRecovered() && featureService.clusterHasFeature(state, f); + }; this.client = client; this.searchUsageHolder = usageService.getSearchUsageHolder(); } @@ -73,7 +85,14 @@ protected void doExecute(Task task, SearchTemplateRequest request, ActionListene final SearchTemplateResponse response = new SearchTemplateResponse(); boolean success = false; try { - SearchRequest searchRequest = convert(request, response, scriptService, xContentRegistry, searchUsageHolder); + SearchRequest searchRequest = convert( + request, + response, + scriptService, + xContentRegistry, + clusterSupportsFeature, + searchUsageHolder + ); if (searchRequest != null) { client.search(searchRequest, listener.delegateResponse((l, e) -> { response.decRef(); @@ -102,6 +121,7 @@ static SearchRequest convert( SearchTemplateResponse response, ScriptService scriptService, NamedXContentRegistry xContentRegistry, + Predicate clusterSupportsFeature, SearchUsageHolder searchUsageHolder ) throws IOException { Script script = new Script( @@ -121,7 +141,7 @@ static SearchRequest convert( XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withRegistry(xContentRegistry) .withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, source)) { - builder.parseXContent(parser, false, searchUsageHolder); + builder.parseXContent(parser, false, searchUsageHolder, clusterSupportsFeature); } if (searchTemplateRequest.isSimulate()) { diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java index 4cd14fa97d710..1efa0ada221ef 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java @@ -28,7 +28,7 @@ public final class RestSearchTemplateActionTests extends RestActionTestCase { @Before public void setUpAction() { - controller().registerHandler(new RestSearchTemplateAction(mock(NamedWriteableRegistry.class))); + controller().registerHandler(new RestSearchTemplateAction(mock(NamedWriteableRegistry.class), nf -> false)); verifyingClient.setExecuteVerifier((actionType, request) -> mock(SearchTemplateResponse.class)); verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(SearchTemplateResponse.class)); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index f7214bfba1eba..d3ed0a66f0abe 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -53,7 +53,7 @@ public List getRestHandlers( Supplier nodesInCluster, Predicate clusterSupportsFeature ) { - return Collections.singletonList(new RestRankEvalAction()); + return Collections.singletonList(new RestRankEvalAction(clusterSupportsFeature)); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 590b6d38af79f..757a891f1c357 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.script.Script; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -30,6 +31,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.function.Predicate; /** * Specification of the ranking evaluation request.
@@ -132,13 +134,13 @@ public void setMaxConcurrentSearches(int maxConcurrentSearches) { private static final ParseField REQUESTS_FIELD = new ParseField("requests"); private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD = new ParseField("max_concurrent_searches"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + private static final ConstructingObjectParser> PARSER = new ConstructingObjectParser<>( "rank_eval", a -> new RankEvalSpec((List) a[0], (EvaluationMetric) a[1], (Collection) a[2]) ); static { - PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> RatedRequest.fromXContent(p), REQUESTS_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> RatedRequest.fromXContent(p, c), REQUESTS_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> parseMetric(p), METRIC_FIELD); PARSER.declareObjectArray( ConstructingObjectParser.optionalConstructorArg(), @@ -156,8 +158,8 @@ private static EvaluationMetric parseMetric(XContentParser parser) throws IOExce return metric; } - public static RankEvalSpec parse(XContentParser parser) { - return PARSER.apply(parser, null); + public static RankEvalSpec parse(XContentParser parser, Predicate clusterSupportsFeature) { + return PARSER.apply(parser, clusterSupportsFeature); } static class ScriptWithId { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index ab5b60655556d..3b199c343ec6c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -31,6 +32,7 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; /** * Definition of a particular query in the ranking evaluation request.
@@ -246,7 +248,7 @@ public void addSummaryFields(List summaryFieldsToAdd) { private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + private static final ConstructingObjectParser> PARSER = new ConstructingObjectParser<>( "request", a -> new RatedRequest( (String) a[0], @@ -262,7 +264,7 @@ public void addSummaryFields(List summaryFieldsToAdd) { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> RatedDocument.fromXContent(p), RATINGS_FIELD); PARSER.declareObject( ConstructingObjectParser.optionalConstructorArg(), - (p, c) -> new SearchSourceBuilder().parseXContent(p, false), + (p, c) -> new SearchSourceBuilder().parseXContent(p, false, c), REQUEST_FIELD ); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), PARAMS_FIELD); @@ -273,8 +275,8 @@ public void addSummaryFields(List summaryFieldsToAdd) { /** * parse from rest representation */ - public static RatedRequest fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); + public static RatedRequest fromXContent(XContentParser parser, Predicate clusterSupportsFeature) { + return PARSER.apply(parser, clusterSupportsFeature); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 150d7053e05eb..dc47d409e4da2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -21,6 +22,7 @@ import java.io.IOException; import java.util.List; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -82,6 +84,12 @@ public class RestRankEvalAction extends BaseRestHandler { public static final String ENDPOINT = "_rank_eval"; + private Predicate clusterSupportsFeature; + + public RestRankEvalAction(Predicate clusterSupportsFeature) { + this.clusterSupportsFeature = clusterSupportsFeature; + } + @Override public List routes() { return List.of( @@ -96,7 +104,7 @@ public List routes() { protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(); try (XContentParser parser = request.contentOrSourceParamParser()) { - parseRankEvalRequest(rankEvalRequest, request, parser); + parseRankEvalRequest(rankEvalRequest, request, parser, clusterSupportsFeature); } return channel -> client.executeLocally( RankEvalPlugin.ACTION, @@ -105,13 +113,18 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli ); } - private static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, XContentParser parser) { + private static void parseRankEvalRequest( + RankEvalRequest rankEvalRequest, + RestRequest request, + XContentParser parser, + Predicate clusterSupportsFeature + ) { rankEvalRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); rankEvalRequest.indicesOptions(IndicesOptions.fromRequest(request, rankEvalRequest.indicesOptions())); if (request.hasParam("search_type")) { rankEvalRequest.searchType(SearchType.fromString(request.param("search_type"))); } - RankEvalSpec spec = RankEvalSpec.parse(parser); + RankEvalSpec spec = RankEvalSpec.parse(parser, clusterSupportsFeature); rankEvalRequest.setRankEvalSpec(spec); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 63995d3b6151e..ad1b54afd985d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -17,11 +17,15 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; @@ -40,6 +44,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.XContentHelper.createParser; import static org.elasticsearch.index.rankeval.RatedRequest.validateEvaluatedQuery; @@ -61,6 +66,7 @@ public class TransportRankEvalAction extends HandledTransportAction clusterSupportsFeature; @Inject public TransportRankEvalAction( @@ -68,11 +74,17 @@ public TransportRankEvalAction( Client client, TransportService transportService, ScriptService scriptService, - NamedXContentRegistry namedXContentRegistry + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + FeatureService featureService ) { super(RankEvalPlugin.ACTION.name(), transportService, actionFilters, RankEvalRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.scriptService = scriptService; this.namedXContentRegistry = namedXContentRegistry; + this.clusterSupportsFeature = f -> { + ClusterState state = clusterService.state(); + return state.clusterRecovered() && featureService.clusterHasFeature(state, f); + }; this.client = client; } @@ -107,7 +119,7 @@ protected void doExecute(Task task, RankEvalRequest request, ActionListener false); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); @@ -132,7 +132,7 @@ public void testXContentParsingIsNotLenient() throws IOException { BytesReference originalBytes = toShuffledXContent(testItem, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()); BytesReference withRandomFields = insertRandomFields(xContentType, originalBytes, null, random()); try (XContentParser parser = createParser(xContentType.xContent(), withRandomFields)) { - Exception exception = expectThrows(Exception.class, () -> RankEvalSpec.parse(parser)); + Exception exception = expectThrows(Exception.class, () -> RankEvalSpec.parse(parser, nf -> false)); assertThat(exception.getMessage(), containsString("[rank_eval] failed to parse field")); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index c5a09d67d94d0..da8872d9b348f 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -115,7 +115,7 @@ public void testXContentRoundtrip() throws IOException { try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); - RatedRequest parsedItem = RatedRequest.fromXContent(itemParser); + RatedRequest parsedItem = RatedRequest.fromXContent(itemParser, nf -> false); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); @@ -128,7 +128,7 @@ public void testXContentParsingIsNotLenient() throws IOException { BytesReference originalBytes = toShuffledXContent(testItem, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()); BytesReference withRandomFields = insertRandomFields(xContentType, originalBytes, null, random()); try (XContentParser parser = createParser(xContentType.xContent(), withRandomFields)) { - Throwable exception = expectThrows(XContentParseException.class, () -> RatedRequest.fromXContent(parser)); + Throwable exception = expectThrows(XContentParseException.class, () -> RatedRequest.fromXContent(parser, nf -> false)); if (exception.getCause() != null) { assertThat(exception.getMessage(), containsString("[request] failed to parse field")); exception = exception.getCause(); @@ -359,7 +359,7 @@ public void testParseFromXContent() throws IOException { ] }"""; try (XContentParser parser = createParser(JsonXContent.jsonXContent, querySpecString)) { - RatedRequest specification = RatedRequest.fromXContent(parser); + RatedRequest specification = RatedRequest.fromXContent(parser, nf -> false); assertEquals("my_qa_query", specification.getId()); assertNotNull(specification.getEvaluationRequest()); List ratedDocs = specification.getRatedDocs(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java index 982d1afcf6dd3..89a850b43e9cd 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java @@ -15,8 +15,10 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.FeatureService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; @@ -81,7 +83,9 @@ public void multiSearch(MultiSearchRequest request, ActionListener clusterSupportsFeature, Map> bodyConsumers ) throws IOException { assert internal != null : "Request should not be null"; @@ -55,7 +58,14 @@ protected void parseInternalRequest( IntConsumer sizeConsumer = restRequest.getRestApiVersion() == RestApiVersion.V_7 ? size -> setMaxDocsFromSearchSize(internal, size) : size -> failOnSizeSpecified(); - RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, namedWriteableRegistry, sizeConsumer); + RestSearchAction.parseSearchRequest( + searchRequest, + restRequest, + parser, + namedWriteableRegistry, + clusterSupportsFeature, + sizeConsumer + ); } searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size())); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java index 88f22478d3cff..8cdfc77db6f7f 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java @@ -76,9 +76,9 @@ public List getRestHandlers( Predicate clusterSupportsFeature ) { return Arrays.asList( - new RestReindexAction(namedWriteableRegistry), - new RestUpdateByQueryAction(namedWriteableRegistry), - new RestDeleteByQueryAction(namedWriteableRegistry), + new RestReindexAction(namedWriteableRegistry, clusterSupportsFeature), + new RestUpdateByQueryAction(namedWriteableRegistry, clusterSupportsFeature), + new RestDeleteByQueryAction(namedWriteableRegistry, clusterSupportsFeature), new RestRethrottleAction(nodesInCluster) ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java index 99bd0c51f3084..cc98dc06575b8 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.rest.RestRequest; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -30,10 +32,12 @@ public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler { private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestDeleteByQueryAction(NamedWriteableRegistry namedWriteableRegistry) { + public RestDeleteByQueryAction(NamedWriteableRegistry namedWriteableRegistry, Predicate clusterSupportsFeature) { super(DeleteByQueryAction.INSTANCE); this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -70,7 +74,7 @@ protected DeleteByQueryRequest buildRequest(RestRequest request, NamedWriteableR consumers.put("conflicts", o -> internal.setConflicts((String) o)); consumers.put("max_docs", s -> setMaxDocsValidateIdentical(internal, ((Number) s).intValue())); - parseInternalRequest(internal, request, namedWriteableRegistry, consumers); + parseInternalRequest(internal, request, namedWriteableRegistry, clusterSupportsFeature, consumers); return internal; } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java index 44cbe4712455f..253fd581cfceb 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.rest.RestRequest; @@ -22,6 +23,7 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import java.util.function.Predicate; import static org.elasticsearch.core.TimeValue.parseTimeValue; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -33,10 +35,12 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler implements RestRequestFilter { private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestReindexAction(NamedWriteableRegistry namedWriteableRegistry) { + public RestReindexAction(NamedWriteableRegistry namedWriteableRegistry, Predicate clusterSupportsFeature) { super(ReindexAction.INSTANCE); this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -64,7 +68,7 @@ protected ReindexRequest buildRequest(RestRequest request, NamedWriteableRegistr ReindexRequest internal; try (XContentParser parser = request.contentParser()) { - internal = ReindexRequest.fromXContent(parser); + internal = ReindexRequest.fromXContent(parser, clusterSupportsFeature); } if (request.hasParam("scroll")) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java index b99e5acbd411d..50536a164727a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.UpdateByQueryAction; import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.rest.RestRequest; @@ -24,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -31,10 +33,12 @@ public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler { private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestUpdateByQueryAction(NamedWriteableRegistry namedWriteableRegistry) { + public RestUpdateByQueryAction(NamedWriteableRegistry namedWriteableRegistry, Predicate clusterSupportsFeature) { super(UpdateByQueryAction.INSTANCE); this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -74,7 +78,7 @@ protected UpdateByQueryRequest buildRequest(RestRequest request, NamedWriteableR consumers.put("script", o -> internal.setScript(Script.parse(o))); consumers.put("max_docs", s -> setMaxDocsValidateIdentical(internal, ((Number) s).intValue())); - parseInternalRequest(internal, request, namedWriteableRegistry, consumers); + parseInternalRequest(internal, request, namedWriteableRegistry, clusterSupportsFeature, consumers); internal.setPipeline(request.param("pipeline")); return internal; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java index 241707f6e0f93..aa457fae9e377 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java @@ -31,7 +31,7 @@ public final class RestDeleteByQueryActionTests extends RestActionTestCase { @Before public void setUpAction() { - controller().registerHandler(new RestDeleteByQueryAction(mock(NamedWriteableRegistry.class))); + controller().registerHandler(new RestDeleteByQueryAction(mock(NamedWriteableRegistry.class), nf -> false)); verifyingClient.setExecuteVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java index 3484b61ca2c9a..ddb8c2ce0225d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java @@ -32,7 +32,7 @@ public class RestReindexActionTests extends RestActionTestCase { @Before public void setUpAction() { - action = new RestReindexAction(mock(NamedWriteableRegistry.class)); + action = new RestReindexAction(mock(NamedWriteableRegistry.class), nf -> false); controller().registerHandler(action); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java index 83e298c3a235f..a3f468df89e1e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java @@ -31,7 +31,7 @@ public final class RestUpdateByQueryActionTests extends RestActionTestCase { @Before public void setUpAction() { - controller().registerHandler(new RestUpdateByQueryAction(mock(NamedWriteableRegistry.class))); + controller().registerHandler(new RestUpdateByQueryAction(mock(NamedWriteableRegistry.class), nf -> false)); verifyingClient.setExecuteVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java index 2277f4415d4db..8a2071584b4a0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java @@ -1265,12 +1265,15 @@ public void testScriptWithValueType() throws Exception { String source = builder.toString(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { - assertNoFailuresAndResponse(prepareSearch("idx").setSource(new SearchSourceBuilder().parseXContent(parser, true)), response -> { - LongTerms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(1)); - }); + assertNoFailuresAndResponse( + prepareSearch("idx").setSource(new SearchSourceBuilder().parseXContent(parser, true, nf -> false)), + response -> { + LongTerms terms = response.getAggregations().get("terms"); + assertThat(terms, notNullValue()); + assertThat(terms.getName(), equalTo("terms")); + assertThat(terms.getBuckets().size(), equalTo(1)); + } + ); } String invalidValueType = source.replaceAll("\"value_type\":\"n.*\"", "\"value_type\":\"foobar\""); @@ -1278,7 +1281,7 @@ public void testScriptWithValueType() throws Exception { try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidValueType)) { XContentParseException ex = expectThrows( XContentParseException.class, - () -> prepareSearch("idx").setSource(new SearchSourceBuilder().parseXContent(parser, true)).get() + () -> prepareSearch("idx").setSource(new SearchSourceBuilder().parseXContent(parser, true, nf -> false)).get() ); assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); assertThat(ex.getCause().getMessage(), containsString("Unknown value type [foobar]")); diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 51939fce0557a..dc3b02872fd83 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -939,12 +939,14 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< registerHandler.accept(new RestBulkAction(settings)); registerHandler.accept(new RestUpdateAction()); - registerHandler.accept(new RestSearchAction(restController.getSearchUsageHolder(), namedWriteableRegistry)); + registerHandler.accept(new RestSearchAction(restController.getSearchUsageHolder(), namedWriteableRegistry, clusterSupportsFeature)); registerHandler.accept(new RestSearchScrollAction()); registerHandler.accept(new RestClearScrollAction()); registerHandler.accept(new RestOpenPointInTimeAction()); registerHandler.accept(new RestClosePointInTimeAction()); - registerHandler.accept(new RestMultiSearchAction(settings, restController.getSearchUsageHolder(), namedWriteableRegistry)); + registerHandler.accept( + new RestMultiSearchAction(settings, restController.getSearchUsageHolder(), namedWriteableRegistry, clusterSupportsFeature) + ); registerHandler.accept(new RestKnnSearchAction()); registerHandler.accept(new RestValidateQueryAction()); diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index 683ec75c57d76..84c5ed826b14d 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.script.Script; @@ -39,6 +40,7 @@ import java.net.URISyntaxException; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; @@ -316,10 +318,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - static final ObjectParser PARSER = new ObjectParser<>("reindex"); + static final ObjectParser> PARSER = new ObjectParser<>("reindex"); static { - ObjectParser.Parser sourceParser = (parser, request, context) -> { + ObjectParser.Parser> sourceParser = (parser, request, context) -> { // Funky hack to work around Search not having a proper ObjectParser and us wanting to extract query if using remote. Map source = parser.map(); String[] indices = extractStringArray(source, "index"); @@ -337,7 +339,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws parser.contentType() ) ) { - request.getSearchRequest().source().parseXContent(innerParser, false); + request.getSearchRequest().source().parseXContent(innerParser, false, context); } }; @@ -349,7 +351,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws destParser.declareString((s, i) -> s.versionType(VersionType.fromString(i)), new ParseField("version_type")); PARSER.declareField(sourceParser, new ParseField("source"), ObjectParser.ValueType.OBJECT); - PARSER.declareField((p, v, c) -> destParser.parse(p, v.getDestination(), c), new ParseField("dest"), ObjectParser.ValueType.OBJECT); + PARSER.declareField( + (p, v, c) -> destParser.parse(p, v.getDestination(), null), + new ParseField("dest"), + ObjectParser.ValueType.OBJECT + ); PARSER.declareInt( ReindexRequest::setMaxDocsValidateIdentical, @@ -370,9 +376,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws PARSER.declareString(ReindexRequest::setConflicts, new ParseField("conflicts")); } - public static ReindexRequest fromXContent(XContentParser parser) throws IOException { + public static ReindexRequest fromXContent(XContentParser parser, Predicate clusterSupportsFeature) throws IOException { ReindexRequest reindexRequest = new ReindexRequest(); - PARSER.parse(parser, reindexRequest, null); + PARSER.parse(parser, reindexRequest, clusterSupportsFeature); return reindexRequest; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 66e7f8cdcbc62..69cc4f23f3956 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Tuple; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -36,6 +37,7 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -50,11 +52,18 @@ public class RestMultiSearchAction extends BaseRestHandler { private final boolean allowExplicitIndex; private final SearchUsageHolder searchUsageHolder; private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestMultiSearchAction(Settings settings, SearchUsageHolder searchUsageHolder, NamedWriteableRegistry namedWriteableRegistry) { + public RestMultiSearchAction( + Settings settings, + SearchUsageHolder searchUsageHolder, + NamedWriteableRegistry namedWriteableRegistry, + Predicate clusterSupportsFeature + ) { this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); this.searchUsageHolder = searchUsageHolder; this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -76,7 +85,13 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final MultiSearchRequest multiSearchRequest = parseRequest(request, namedWriteableRegistry, allowExplicitIndex, searchUsageHolder); + final MultiSearchRequest multiSearchRequest = parseRequest( + request, + namedWriteableRegistry, + allowExplicitIndex, + searchUsageHolder, + clusterSupportsFeature + ); return channel -> { final RestCancellableNodeClient cancellableClient = new RestCancellableNodeClient(client, request.getHttpChannel()); cancellableClient.execute( @@ -94,9 +109,17 @@ public static MultiSearchRequest parseRequest( RestRequest restRequest, NamedWriteableRegistry namedWriteableRegistry, boolean allowExplicitIndex, - SearchUsageHolder searchUsageHolder + SearchUsageHolder searchUsageHolder, + Predicate clusterSupportsFeature ) throws IOException { - return parseRequest(restRequest, namedWriteableRegistry, allowExplicitIndex, searchUsageHolder, (k, v, r) -> false); + return parseRequest( + restRequest, + namedWriteableRegistry, + allowExplicitIndex, + searchUsageHolder, + clusterSupportsFeature, + (k, v, r) -> false + ); } /** @@ -108,6 +131,7 @@ public static MultiSearchRequest parseRequest( NamedWriteableRegistry namedWriteableRegistry, boolean allowExplicitIndex, SearchUsageHolder searchUsageHolder, + Predicate clusterSupportsFeature, TriFunction extraParamParser ) throws IOException { if (restRequest.getRestApiVersion() == RestApiVersion.V_7 && restRequest.hasParam("type")) { @@ -136,7 +160,7 @@ public static MultiSearchRequest parseRequest( } parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, parser) -> { - searchRequest.source(new SearchSourceBuilder().parseXContent(parser, false, searchUsageHolder)); + searchRequest.source(new SearchSourceBuilder().parseXContent(parser, false, searchUsageHolder, clusterSupportsFeature)); RestSearchAction.validateSearchRequest(restRequest, searchRequest); if (searchRequest.pointInTimeBuilder() != null) { RestSearchAction.preparePointInTime(searchRequest, restRequest); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 05c56da2cda48..cfb70da9fb454 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -47,6 +48,7 @@ import java.util.Locale; import java.util.Set; import java.util.function.IntConsumer; +import java.util.function.Predicate; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.core.TimeValue.parseTimeValue; @@ -70,10 +72,16 @@ public class RestSearchAction extends BaseRestHandler { private final SearchUsageHolder searchUsageHolder; private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestSearchAction(SearchUsageHolder searchUsageHolder, NamedWriteableRegistry namedWriteableRegistry) { + public RestSearchAction( + SearchUsageHolder searchUsageHolder, + NamedWriteableRegistry namedWriteableRegistry, + Predicate clusterSupportsFeature + ) { this.searchUsageHolder = searchUsageHolder; this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -116,7 +124,15 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC */ IntConsumer setSize = size -> searchRequest.source().size(size); request.withContentOrSourceParamParserOrNull( - parser -> parseSearchRequest(searchRequest, request, parser, namedWriteableRegistry, setSize, searchUsageHolder) + parser -> parseSearchRequest( + searchRequest, + request, + parser, + namedWriteableRegistry, + clusterSupportsFeature, + setSize, + searchUsageHolder + ) ); return channel -> { @@ -133,6 +149,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC * @param requestContentParser body of the request to read. This method does not attempt to read the body from the {@code request} * parameter * @param namedWriteableRegistry the registry of named writeables + * @param clusterSupportsFeature used to check if certain features are available in this cluster * @param setSize how the size url parameter is handled. {@code udpate_by_query} and regular search differ here. */ public static void parseSearchRequest( @@ -140,9 +157,10 @@ public static void parseSearchRequest( RestRequest request, XContentParser requestContentParser, NamedWriteableRegistry namedWriteableRegistry, + Predicate clusterSupportsFeature, IntConsumer setSize ) throws IOException { - parseSearchRequest(searchRequest, request, requestContentParser, namedWriteableRegistry, setSize, null); + parseSearchRequest(searchRequest, request, requestContentParser, namedWriteableRegistry, clusterSupportsFeature, setSize, null); } /** @@ -153,6 +171,7 @@ public static void parseSearchRequest( * @param requestContentParser body of the request to read. This method does not attempt to read the body from the {@code request} * parameter, will be null when there is no request body to parse * @param namedWriteableRegistry the registry of named writeables + @param clusterSupportsFeature used to check if certain features are available in this cluster * @param setSize how the size url parameter is handled. {@code udpate_by_query} and regular search differ here. * @param searchUsageHolder the holder of search usage stats */ @@ -161,6 +180,7 @@ public static void parseSearchRequest( RestRequest request, @Nullable XContentParser requestContentParser, NamedWriteableRegistry namedWriteableRegistry, + Predicate clusterSupportsFeature, IntConsumer setSize, @Nullable SearchUsageHolder searchUsageHolder ) throws IOException { @@ -175,9 +195,9 @@ public static void parseSearchRequest( searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); if (requestContentParser != null) { if (searchUsageHolder == null) { - searchRequest.source().parseXContent(requestContentParser, true); + searchRequest.source().parseXContent(requestContentParser, true, clusterSupportsFeature); } else { - searchRequest.source().parseXContent(requestContentParser, true, searchUsageHolder); + searchRequest.source().parseXContent(requestContentParser, true, searchUsageHolder, clusterSupportsFeature); } } diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 56b888138d8e8..649c40c856fe8 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -21,6 +21,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; @@ -63,6 +64,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.Collectors; @@ -1238,10 +1240,15 @@ private SearchSourceBuilder shallowCopy( * @param parser The xContent parser. * @param checkTrailingTokens If true throws a parsing exception when extra tokens are found after the main object. * @param searchUsageHolder holder for the search usage statistics + * @param clusterSupportsFeature used to check if certain features are available on this cluster */ - public SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTrailingTokens, SearchUsageHolder searchUsageHolder) - throws IOException { - return parseXContent(parser, checkTrailingTokens, searchUsageHolder::updateUsage); + public SearchSourceBuilder parseXContent( + XContentParser parser, + boolean checkTrailingTokens, + SearchUsageHolder searchUsageHolder, + Predicate clusterSupportsFeature + ) throws IOException { + return parseXContent(parser, checkTrailingTokens, searchUsageHolder::updateUsage, clusterSupportsFeature); } /** @@ -1251,13 +1258,22 @@ public SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTra * * @param parser The xContent parser. * @param checkTrailingTokens If true throws a parsing exception when extra tokens are found after the main object. - */ - public SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTrailingTokens) throws IOException { - return parseXContent(parser, checkTrailingTokens, s -> {}); - } - - private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTrailingTokens, Consumer searchUsageConsumer) - throws IOException { + * @param clusterSupportsFeature used to check if certain features are available on this cluster + */ + public SearchSourceBuilder parseXContent( + XContentParser parser, + boolean checkTrailingTokens, + Predicate clusterSupportsFeature + ) throws IOException { + return parseXContent(parser, checkTrailingTokens, s -> {}, clusterSupportsFeature); + } + + private SearchSourceBuilder parseXContent( + XContentParser parser, + boolean checkTrailingTokens, + Consumer searchUsageConsumer, + Predicate clusterSupportsFeature + ) throws IOException { XContentParser.Token token = parser.currentToken(); String currentFieldName = null; if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) { @@ -1588,7 +1604,9 @@ private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTr throw new ParsingException(parser.getTokenLocation(), "Unexpected token [" + token + "] found after the main object."); } } + knnSearch = knnBuilders.stream().map(knnBuilder -> knnBuilder.build(size())).collect(Collectors.toList()); + searchUsageConsumer.accept(searchUsage); return this; } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 4e9c854605f65..501ecceba3acb 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -149,7 +149,7 @@ public KnnSearchBuilder(String field, QueryVectorBuilder queryVectorBuilder, int ); } - private KnnSearchBuilder( + public KnnSearchBuilder( String field, float[] queryVector, QueryVectorBuilder queryVectorBuilder, diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index e9330c9643318..5f24f72d5cc8f 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -97,7 +97,7 @@ public void testFailWithUnknownKey() { ).build(); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> RestMultiSearchAction.parseRequest(restRequest, null, true, new UsageService().getSearchUsageHolder()) + () -> RestMultiSearchAction.parseRequest(restRequest, null, true, new UsageService().getSearchUsageHolder(), nf -> false) ); assertEquals("key [unknown_key] is not supported in the metadata section", ex.getMessage()); } @@ -111,7 +111,13 @@ public void testSimpleAddWithCarriageReturn() throws Exception { new BytesArray(requestContent), XContentType.JSON ).build(); - MultiSearchRequest request = RestMultiSearchAction.parseRequest(restRequest, null, true, new UsageService().getSearchUsageHolder()); + MultiSearchRequest request = RestMultiSearchAction.parseRequest( + restRequest, + null, + true, + new UsageService().getSearchUsageHolder(), + nf -> false + ); assertThat(request.requests().size(), equalTo(1)); assertThat(request.requests().get(0).indices()[0], equalTo("test")); assertThat( @@ -129,7 +135,13 @@ public void testDefaultIndicesOptions() throws IOException { new BytesArray(requestContent), XContentType.JSON ).withParams(Collections.singletonMap("ignore_unavailable", "true")).build(); - MultiSearchRequest request = RestMultiSearchAction.parseRequest(restRequest, null, true, new UsageService().getSearchUsageHolder()); + MultiSearchRequest request = RestMultiSearchAction.parseRequest( + restRequest, + null, + true, + new UsageService().getSearchUsageHolder(), + nf -> false + ); assertThat(request.requests().size(), equalTo(1)); assertThat(request.requests().get(0).indices()[0], equalTo("test")); assertThat( @@ -238,7 +250,7 @@ public void testMsearchTerminatedByNewline() throws Exception { ).build(); IllegalArgumentException expectThrows = expectThrows( IllegalArgumentException.class, - () -> RestMultiSearchAction.parseRequest(restRequest, null, true, new UsageService().getSearchUsageHolder()) + () -> RestMultiSearchAction.parseRequest(restRequest, null, true, new UsageService().getSearchUsageHolder(), nf -> false) ); assertEquals("The msearch request must be terminated by a newline [\n]", expectThrows.getMessage()); @@ -251,7 +263,8 @@ public void testMsearchTerminatedByNewline() throws Exception { restRequestWithNewLine, null, true, - new UsageService().getSearchUsageHolder() + new UsageService().getSearchUsageHolder(), + nf -> false ); assertEquals(3, msearchRequest.requests().size()); } @@ -268,7 +281,9 @@ private MultiSearchRequest parseMultiSearchRequest(RestRequest restRequest) thro MultiSearchRequest request = new MultiSearchRequest(); RestMultiSearchAction.parseMultiLineRequest(restRequest, SearchRequest.DEFAULT_INDICES_OPTIONS, true, (searchRequest, parser) -> { - searchRequest.source(new SearchSourceBuilder().parseXContent(parser, false, new UsageService().getSearchUsageHolder())); + searchRequest.source( + new SearchSourceBuilder().parseXContent(parser, false, new UsageService().getSearchUsageHolder(), nf -> false) + ); request.add(searchRequest); }); return request; @@ -318,7 +333,8 @@ public void testMultiLineSerialization() throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent( p, false, - new UsageService().getSearchUsageHolder() + new UsageService().getSearchUsageHolder(), + nf -> false ); if (searchSourceBuilder.equals(new SearchSourceBuilder()) == false) { r.source(searchSourceBuilder); diff --git a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java index 337b6effc3252..65c060aa9005a 100644 --- a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java @@ -115,7 +115,7 @@ protected ReindexRequest createTestInstance() { @Override protected ReindexRequest doParseInstance(XContentParser parser) throws IOException { - return ReindexRequest.fromXContent(parser); + return ReindexRequest.fromXContent(parser, nf -> false); } @Override @@ -349,7 +349,7 @@ public void testReindexFromRemoteRequestParsing() throws IOException { request = BytesReference.bytes(b); } try (XContentParser p = createParser(JsonXContent.jsonXContent, request)) { - ReindexRequest r = ReindexRequest.fromXContent(p); + ReindexRequest r = ReindexRequest.fromXContent(p, nf -> false); assertEquals("localhost", r.getRemoteInfo().getHost()); assertArrayEquals(new String[] { "source" }, r.getSearchRequest().indices()); } @@ -403,7 +403,7 @@ private ReindexRequest parseRequestWithSourceIndices(Object sourceIndices) throw request = BytesReference.bytes(b); } try (XContentParser p = createParser(JsonXContent.jsonXContent, request)) { - return ReindexRequest.fromXContent(p); + return ReindexRequest.fromXContent(p, nf -> false); } } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java index 7ad935744680f..f2a11336c7f4b 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java @@ -34,7 +34,12 @@ public final class RestMultiSearchActionTests extends RestActionTestCase { @Before public void setUpAction() { - action = new RestMultiSearchAction(Settings.EMPTY, new UsageService().getSearchUsageHolder(), mock(NamedWriteableRegistry.class)); + action = new RestMultiSearchAction( + Settings.EMPTY, + new UsageService().getSearchUsageHolder(), + mock(NamedWriteableRegistry.class), + nf -> false + ); controller().registerHandler(action); verifyingClient.setExecuteVerifier((actionType, request) -> mock(MultiSearchResponse.class)); verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(MultiSearchResponse.class)); diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java index 6c1a234b32cd9..5f641ef8fd84f 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java @@ -37,7 +37,7 @@ public final class RestSearchActionTests extends RestActionTestCase { @Before public void setUpAction() { - action = new RestSearchAction(new UsageService().getSearchUsageHolder(), mock(NamedWriteableRegistry.class)); + action = new RestSearchAction(new UsageService().getSearchUsageHolder(), mock(NamedWriteableRegistry.class), nf -> false); controller().registerHandler(action); verifyingClient.setExecuteVerifier((actionType, request) -> mock(SearchResponse.class)); verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(SearchResponse.class)); diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index ee7cf7a60d639..8ee1c64ddbb22 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -93,13 +93,16 @@ public void testFromXContentInvalid() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{}}")) { XContentParseException exc = expectThrows( XContentParseException.class, - () -> new SearchSourceBuilder().parseXContent(parser, true) + () -> new SearchSourceBuilder().parseXContent(parser, true, nf -> false) ); assertThat(exc.getMessage(), containsString("Unexpected close marker")); } try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{}{}")) { - ParsingException exc = expectThrows(ParsingException.class, () -> new SearchSourceBuilder().parseXContent(parser, true)); + ParsingException exc = expectThrows( + ParsingException.class, + () -> new SearchSourceBuilder().parseXContent(parser, true, nf -> false) + ); assertThat(exc.getDetailedMessage(), containsString("found after the main object")); } } @@ -109,7 +112,7 @@ private static void assertParseSearchSource(SearchSourceBuilder testBuilder, XCo parser.nextToken(); // sometimes we move it on the START_OBJECT to // test the embedded case } - SearchSourceBuilder newBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder newBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertNull(parser.nextToken()); assertEquals(testBuilder, newBuilder); assertEquals(testBuilder.hashCode(), newBuilder.hashCode()); @@ -150,7 +153,7 @@ public void testParseIncludeExclude() throws IOException { { "_source": { "includes": "include", "excludes": "*.field2"}} """; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertArrayEquals(new String[] { "*.field2" }, searchSourceBuilder.fetchSource().excludes()); assertArrayEquals(new String[] { "include" }, searchSourceBuilder.fetchSource().includes()); } @@ -158,7 +161,7 @@ public void testParseIncludeExclude() throws IOException { { String restContent = " { \"_source\": false}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertArrayEquals(new String[] {}, searchSourceBuilder.fetchSource().excludes()); assertArrayEquals(new String[] {}, searchSourceBuilder.fetchSource().includes()); assertFalse(searchSourceBuilder.fetchSource().fetchSource()); @@ -180,7 +183,10 @@ public void testMultipleQueryObjectsAreRejected() throws Exception { } } }""".indent(1); try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - ParsingException e = expectThrows(ParsingException.class, () -> new SearchSourceBuilder().parseXContent(parser, true)); + ParsingException e = expectThrows( + ParsingException.class, + () -> new SearchSourceBuilder().parseXContent(parser, true, nf -> false) + ); assertEquals("[multi_match] malformed query, expected [END_OBJECT] but found [FIELD_NAME]", e.getMessage()); } } @@ -226,7 +232,7 @@ public void testParseAndRewrite() throws IOException { } }"""; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertThat(searchSourceBuilder.query(), instanceOf(BoolQueryBuilder.class)); assertThat(searchSourceBuilder.rescores().get(0), instanceOf(QueryRescorerBuilder.class)); assertThat( @@ -252,7 +258,7 @@ public void testParseSort() throws IOException { { String restContent = " { \"sort\": \"foo\"}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); searchSourceBuilder = rewrite(searchSourceBuilder); assertEquals(1, searchSourceBuilder.sorts().size()); assertEquals(new FieldSortBuilder("foo"), searchSourceBuilder.sorts().get(0)); @@ -269,7 +275,7 @@ public void testParseSort() throws IOException { "_score" ]}"""; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); searchSourceBuilder = rewrite(searchSourceBuilder); assertEquals(5, searchSourceBuilder.sorts().size()); assertEquals(new FieldSortBuilder("post_date"), searchSourceBuilder.sorts().get(0)); @@ -294,7 +300,7 @@ public void testAggsParsing() throws IOException { } """; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); searchSourceBuilder = rewrite(searchSourceBuilder); assertEquals(1, searchSourceBuilder.aggregations().count()); } @@ -311,7 +317,7 @@ public void testAggsParsing() throws IOException { } """; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); searchSourceBuilder = rewrite(searchSourceBuilder); assertEquals(1, searchSourceBuilder.aggregations().count()); } @@ -339,7 +345,7 @@ public void testParseRescore() throws IOException { } """; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); searchSourceBuilder = rewrite(searchSourceBuilder); assertEquals(1, searchSourceBuilder.rescores().size()); assertEquals( @@ -366,7 +372,7 @@ public void testParseRescore() throws IOException { } """; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); searchSourceBuilder = rewrite(searchSourceBuilder); assertEquals(1, searchSourceBuilder.rescores().size()); assertEquals( @@ -383,7 +389,7 @@ public void testTimeoutWithUnits() throws IOException { { "query": { "match_all": {}}, "timeout": "%s"} """, timeout); try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) { - final SearchSourceBuilder builder = new SearchSourceBuilder().parseXContent(parser, true); + final SearchSourceBuilder builder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertThat(builder.timeout(), equalTo(TimeValue.parseTimeValue(timeout, null, "timeout"))); } } @@ -396,7 +402,7 @@ public void testTimeoutWithoutUnits() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) { final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new SearchSourceBuilder().parseXContent(parser, true) + () -> new SearchSourceBuilder().parseXContent(parser, true, nf -> false) ); assertThat(e, hasToString(containsString("unit is missing or unrecognized"))); } @@ -452,7 +458,7 @@ public void testParseIndicesBoost() throws IOException { String restContent = """ { "indices_boost": {"foo": 1.0, "bar": 2.0}}"""; try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, restContent, RestApiVersion.V_7)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertEquals(2, searchSourceBuilder.indexBoosts().size()); assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0)); assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1)); @@ -466,7 +472,7 @@ public void testParseIndicesBoost() throws IOException { "indices_boost": [ { "foo": 1 }, { "bar": 2 }, { "baz": 3 } ] }"""; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertEquals(3, searchSourceBuilder.indexBoosts().size()); assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0)); assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1)); @@ -534,14 +540,14 @@ public void testNegativeSizeErrors() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> new SearchSourceBuilder().parseXContent(parser, true) + () -> new SearchSourceBuilder().parseXContent(parser, true, nf -> false) ); assertThat(ex.getMessage(), containsString(Integer.toString(boundedRandomSize))); } restContent = "{\"size\" : -1}"; try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, restContent, RestApiVersion.V_7)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); assertEquals(-1, searchSourceBuilder.size()); } assertCriticalWarnings( @@ -562,7 +568,7 @@ public void testNegativeTerminateAfter() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> new SearchSourceBuilder().parseXContent(parser, true) + () -> new SearchSourceBuilder().parseXContent(parser, true, nf -> false) ); assertThat(ex.getMessage(), containsString("terminateAfter must be > 0")); } @@ -580,7 +586,7 @@ public void testNegativeTrackTotalHits() throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> new SearchSourceBuilder().parseXContent(parser, true) + () -> new SearchSourceBuilder().parseXContent(parser, true, nf -> false) ); assertEquals("[track_total_hits] parameter must be positive or equals to -1, got " + randomNegativeValue, ex.getMessage()); } @@ -709,7 +715,7 @@ public void testEmptySectionsAreNotTracked() throws IOException { private void assertSectionNotTracked(SearchUsageHolder searchUsageHolder, String request) throws IOException { long totalSearch = searchUsageHolder.getSearchUsageStats().getTotalSearchCount(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, request)) { - new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder); + new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder, nf -> false); assertEquals(totalSearch + 1, searchUsageHolder.getSearchUsageStats().getTotalSearchCount()); assertEquals(0, searchUsageHolder.getSearchUsageStats().getSectionsUsage().size()); } @@ -718,7 +724,7 @@ private void assertSectionNotTracked(SearchUsageHolder searchUsageHolder, String private void assertParseFailureNotTracked(SearchUsageHolder searchUsageHolder, String request) throws IOException { long totalSearch = searchUsageHolder.getSearchUsageStats().getTotalSearchCount(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, request)) { - expectThrows(Exception.class, () -> new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder)); + expectThrows(Exception.class, () -> new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder, nf -> false)); assertEquals(totalSearch, searchUsageHolder.getSearchUsageStats().getTotalSearchCount()); assertEquals(0, searchUsageHolder.getSearchUsageStats().getSectionsUsage().size()); } @@ -789,7 +795,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) { assertEquals(0, searchUsageHolder.getSearchUsageStats().getTotalSearchCount()); try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(searchSourceBuilder))) { - new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder); + new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder, nf -> false); } SearchUsageStats searchUsageStats = searchUsageHolder.getSearchUsageStats(); @@ -842,7 +848,7 @@ public void testOnlyQueriesAreTracked() throws IOException { assertEquals(0, searchUsageHolder.getSearchUsageStats().getTotalSearchCount()); try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(searchSourceBuilder))) { - new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder); + new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder, nf -> false); } SearchUsageStats searchUsageStats = searchUsageHolder.getSearchUsageStats(); @@ -865,7 +871,7 @@ public void testSearchTotalUsageCollection() throws IOException { int iters = randomIntBetween(10, 100); for (int i = 0; i < iters; i++) { try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(searchSourceBuilder))) { - new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder); + new SearchSourceBuilder().parseXContent(parser, true, searchUsageHolder, nf -> false); } } @@ -969,7 +975,7 @@ private void assertIndicesBoostParseErrorMessage(String restContent, String expe try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { ParsingException e = expectThrows( ParsingException.class, - () -> new SearchSourceBuilder().parseXContent(parser, true, new UsageService().getSearchUsageHolder()) + () -> new SearchSourceBuilder().parseXContent(parser, true, new UsageService().getSearchUsageHolder(), nf -> false) ); assertEquals(expectedErrorMessage, e.getMessage()); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index 0f80df5e6db50..c551312f68c0b 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -57,7 +57,7 @@ public List getRestHandlers( Predicate clusterSupportsFeature ) { return Arrays.asList( - new RestSubmitAsyncSearchAction(restController.getSearchUsageHolder(), namedWriteableRegistry), + new RestSubmitAsyncSearchAction(restController.getSearchUsageHolder(), namedWriteableRegistry, clusterSupportsFeature), new RestGetAsyncSearchAction(), new RestGetAsyncStatusAction(), new RestDeleteAsyncSearchAction() diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java index f88207343bd60..d98677d456b90 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; @@ -25,6 +26,7 @@ import java.util.List; import java.util.Set; import java.util.function.IntConsumer; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.action.search.RestSearchAction.parseSearchRequest; @@ -36,10 +38,16 @@ public final class RestSubmitAsyncSearchAction extends BaseRestHandler { private final SearchUsageHolder searchUsageHolder; private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestSubmitAsyncSearchAction(SearchUsageHolder searchUsageHolder, NamedWriteableRegistry namedWriteableRegistry) { + public RestSubmitAsyncSearchAction( + SearchUsageHolder searchUsageHolder, + NamedWriteableRegistry namedWriteableRegistry, + Predicate clusterSupportsFeature + ) { this.searchUsageHolder = searchUsageHolder; this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -61,7 +69,15 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli // them as supported. We rely on SubmitAsyncSearchRequest#validate to fail in case they are set. // Note that ccs_minimize_roundtrips is also set this way, which is a supported option. request.withContentOrSourceParamParserOrNull( - parser -> parseSearchRequest(submit.getSearchRequest(), request, parser, namedWriteableRegistry, setSize, searchUsageHolder) + parser -> parseSearchRequest( + submit.getSearchRequest(), + request, + parser, + namedWriteableRegistry, + clusterSupportsFeature, + setSize, + searchUsageHolder + ) ); if (request.hasParam("wait_for_completion_timeout")) { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchActionTests.java index c319ed99c1841..2c9708a930186 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchActionTests.java @@ -37,7 +37,11 @@ public class RestSubmitAsyncSearchActionTests extends RestActionTestCase { @Before public void setUpAction() { - action = new RestSubmitAsyncSearchAction(new UsageService().getSearchUsageHolder(), mock(NamedWriteableRegistry.class)); + action = new RestSubmitAsyncSearchAction( + new UsageService().getSearchUsageHolder(), + mock(NamedWriteableRegistry.class), + nf -> false + ); controller().registerHandler(action); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationTemplateService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationTemplateService.java index bed5cc0cacd6c..f4e5a60618f0a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationTemplateService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationTemplateService.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; @@ -27,15 +28,22 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; public class SearchApplicationTemplateService { private final ScriptService scriptService; private final NamedXContentRegistry xContentRegistry; + private final Predicate clusterSupportsFeature; - public SearchApplicationTemplateService(ScriptService scriptService, NamedXContentRegistry xContentRegistry) { + public SearchApplicationTemplateService( + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Predicate clusterSupportsFeature + ) { this.scriptService = scriptService; this.xContentRegistry = xContentRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } public SearchSourceBuilder renderQuery(SearchApplication searchApplication, Map templateParams) throws IOException, @@ -57,7 +65,7 @@ public SearchSourceBuilder renderQuery(SearchApplication searchApplication, Map< .withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, requestSource)) { SearchSourceBuilder builder = SearchSourceBuilder.searchSource(); - builder.parseXContent(parser, false); + builder.parseXContent(parser, false, clusterSupportsFeature); return builder; } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java index 89e670f9b1de9..c8a2d22f5416e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java @@ -14,11 +14,14 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.script.ScriptService; @@ -29,6 +32,8 @@ import org.elasticsearch.xpack.application.search.SearchApplicationIndexService; import org.elasticsearch.xpack.application.search.SearchApplicationTemplateService; +import java.util.function.Predicate; + public class TransportQuerySearchApplicationAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportQuerySearchApplicationAction.class); @@ -46,7 +51,8 @@ public TransportQuerySearchApplicationAction( NamedWriteableRegistry namedWriteableRegistry, BigArrays bigArrays, ScriptService scriptService, - NamedXContentRegistry xContentRegistry + NamedXContentRegistry xContentRegistry, + FeatureService featureService ) { super( QuerySearchApplicationAction.NAME, @@ -56,7 +62,11 @@ public TransportQuerySearchApplicationAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = client; - this.templateService = new SearchApplicationTemplateService(scriptService, xContentRegistry); + Predicate clusterSupportsFeature = f -> { + ClusterState state = clusterService.state(); + return state.clusterRecovered() && featureService.clusterHasFeature(state, f); + }; + this.templateService = new SearchApplicationTemplateService(scriptService, xContentRegistry, clusterSupportsFeature); this.systemIndexService = new SearchApplicationIndexService(client, clusterService, namedWriteableRegistry, bigArrays); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java index 4a028a5558e87..06a57d2cd2bae 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java @@ -11,11 +11,14 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.script.ScriptService; @@ -27,6 +30,7 @@ import org.elasticsearch.xpack.application.search.SearchApplicationTemplateService; import java.util.Map; +import java.util.function.Predicate; public class TransportRenderSearchApplicationQueryAction extends HandledTransportAction< SearchApplicationSearchRequest, @@ -47,7 +51,8 @@ public TransportRenderSearchApplicationQueryAction( NamedWriteableRegistry namedWriteableRegistry, BigArrays bigArrays, ScriptService scriptService, - NamedXContentRegistry xContentRegistry + NamedXContentRegistry xContentRegistry, + FeatureService featureService ) { super( RenderSearchApplicationQueryAction.NAME, @@ -57,7 +62,11 @@ public TransportRenderSearchApplicationQueryAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.systemIndexService = new SearchApplicationIndexService(client, clusterService, namedWriteableRegistry, bigArrays); - this.templateService = new SearchApplicationTemplateService(scriptService, xContentRegistry); + Predicate clusterSupportsFeature = f -> { + ClusterState state = clusterService.state(); + return state.clusterRecovered() && featureService.clusterHasFeature(state, f); + }; + this.templateService = new SearchApplicationTemplateService(scriptService, xContentRegistry, clusterSupportsFeature); } @Override diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 724482959e3c9..c2e4e2aa2ca98 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -367,8 +367,8 @@ public List getRestHandlers( ) { return List.of( new RestGetGlobalCheckpointsAction(), - new RestFleetSearchAction(restController.getSearchUsageHolder(), namedWriteableRegistry), - new RestFleetMultiSearchAction(settings, restController.getSearchUsageHolder(), namedWriteableRegistry), + new RestFleetSearchAction(restController.getSearchUsageHolder(), namedWriteableRegistry, clusterSupportsFeature), + new RestFleetMultiSearchAction(settings, restController.getSearchUsageHolder(), namedWriteableRegistry, clusterSupportsFeature), new RestGetSecretsAction(), new RestPostSecretsAction(), new RestDeleteSecretsAction() diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetMultiSearchAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetMultiSearchAction.java index 5e7ef365b6592..28cc7c5172631 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetMultiSearchAction.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetMultiSearchAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeTimeValue; @@ -42,15 +44,18 @@ public class RestFleetMultiSearchAction extends BaseRestHandler { private final boolean allowExplicitIndex; private final SearchUsageHolder searchUsageHolder; private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; public RestFleetMultiSearchAction( Settings settings, SearchUsageHolder searchUsageHolder, - NamedWriteableRegistry namedWriteableRegistry + NamedWriteableRegistry namedWriteableRegistry, + Predicate clusterSupportsFeature ) { this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); this.searchUsageHolder = searchUsageHolder; this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -75,6 +80,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli namedWriteableRegistry, allowExplicitIndex, searchUsageHolder, + clusterSupportsFeature, (key, value, searchRequest) -> { if ("wait_for_checkpoints".equals(key)) { String[] stringWaitForCheckpoints = nodeStringArrayValue(value); diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java index ce606fdd17363..e1281f4f20a4c 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -29,6 +30,7 @@ import java.util.List; import java.util.Set; import java.util.function.IntConsumer; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -38,10 +40,16 @@ public class RestFleetSearchAction extends BaseRestHandler { private final SearchUsageHolder searchUsageHolder; private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestFleetSearchAction(SearchUsageHolder searchUsageHolder, NamedWriteableRegistry namedWriteableRegistry) { + public RestFleetSearchAction( + SearchUsageHolder searchUsageHolder, + NamedWriteableRegistry namedWriteableRegistry, + Predicate clusterSupportsFeature + ) { this.searchUsageHolder = searchUsageHolder; this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -71,7 +79,15 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli IntConsumer setSize = size -> searchRequest.source().size(size); request.withContentOrSourceParamParserOrNull(parser -> { - RestSearchAction.parseSearchRequest(searchRequest, request, parser, namedWriteableRegistry, setSize, searchUsageHolder); + RestSearchAction.parseSearchRequest( + searchRequest, + request, + parser, + namedWriteableRegistry, + clusterSupportsFeature, + setSize, + searchUsageHolder + ); String[] stringWaitForCheckpoints = request.paramAsStringArray("wait_for_checkpoints", Strings.EMPTY_ARRAY); final long[] waitForCheckpoints = new long[stringWaitForCheckpoints.length]; for (int i = 0; i < stringWaitForCheckpoints.length; ++i) { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index 35c2ba0e68080..7a96bec42eb7b 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -14,6 +14,7 @@ dependencies { javaRestTestImplementation project(path: xpackModule('slm')) javaRestTestImplementation project(path: xpackModule('monitoring')) javaRestTestImplementation project(path: xpackModule('transform')) + javaRestTestImplementation project(path: xpackModule('rank-rrf')) } // location for keys and certificates diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java index 44504cd84127a..135f8907faa9b 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java @@ -11,6 +11,7 @@ import org.elasticsearch.license.License; import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rank.RankShardResult; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -18,7 +19,7 @@ import java.util.List; -public class RRFRankPlugin extends Plugin { +public class RRFRankPlugin extends Plugin implements SearchPlugin { public static final LicensedFeature.Momentary RANK_RRF_FEATURE = LicensedFeature.momentary( null, diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/license/100_license.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/license/100_license.yml index 703556a49017e..fb74c935c774c 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/license/100_license.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/license/100_license.yml @@ -75,4 +75,3 @@ setup: - match: { status: 403 } - match: { error.type: security_exception } - match: { error.reason: "current license is non-compliant for [Reciprocal Rank Fusion (RRF)]" } - diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml index 1445dc47963eb..56cb8dd94de0d 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml @@ -1,6 +1,5 @@ setup: - skip: - features: close_to version: ' - 8.7.99' reason: 'rank added in 8.8' @@ -61,7 +60,7 @@ setup: index: test body: track_total_hits: false - fields: [ "text" ] + fields: [ "text", "keyword" ] knn: field: vector query_vector: [0.0] @@ -79,14 +78,17 @@ setup: - match: { hits.hits.0._id: "1" } - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term term" } + - match: { hits.hits.0.fields.keyword.0: "other" } - match: { hits.hits.1._id: "3" } - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term" } + - match: { hits.hits.1.fields.keyword.0: "keyword" } - match: { hits.hits.2._id: "2" } - match: { hits.hits.2._rank: 3 } - match: { hits.hits.2.fields.text.0: "other" } + - match: { hits.hits.2.fields.keyword.0: "other" } --- "Simple rank with multiple bm25 sub searches": @@ -96,7 +98,7 @@ setup: index: test body: track_total_hits: true - fields: [ "text" ] + fields: [ "text", "keyword" ] sub_searches: [ { "query": { @@ -124,10 +126,12 @@ setup: - match: { hits.hits.0._id: "3" } - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } + - match: { hits.hits.0.fields.keyword.0: "keyword" } - match: { hits.hits.1._id: "1" } - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term term" } + - match: { hits.hits.1.fields.keyword.0: "other" } --- "Simple rank with multiple bm25 sub_searches and a knn search": @@ -137,7 +141,7 @@ setup: index: test body: track_total_hits: true - fields: [ "text" ] + fields: [ "text", "keyword" ] knn: field: vector query_vector: [ 0.0 ] @@ -170,11 +174,14 @@ setup: - match: { hits.hits.0._id: "3" } - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } + - match: { hits.hits.0.fields.keyword.0: "keyword" } - match: { hits.hits.1._id: "1" } - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term term" } + - match: { hits.hits.1.fields.keyword.0: "other" } - match: { hits.hits.2._id: "2" } - match: { hits.hits.2._rank: 3 } - match: { hits.hits.2.fields.text.0: "other" } + - match: { hits.hits.2.fields.keyword.0: "other" } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java index 41cca2a219ff4..aef7a266fff37 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java @@ -99,7 +99,7 @@ public List getRestHandlers( Predicate clusterSupportsFeature ) { return Arrays.asList( - new RestRollupSearchAction(namedWriteableRegistry), + new RestRollupSearchAction(namedWriteableRegistry, clusterSupportsFeature), new RestPutRollupJobAction(), new RestStartRollupJobAction(), new RestStopRollupJobAction(), diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java index 68c8fba19af4e..2e02f1d12fb69 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; @@ -18,6 +19,7 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -27,9 +29,11 @@ public class RestRollupSearchAction extends BaseRestHandler { private static final Set RESPONSE_PARAMS = Set.of(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM); private final NamedWriteableRegistry namedWriteableRegistry; + private final Predicate clusterSupportsFeature; - public RestRollupSearchAction(NamedWriteableRegistry namedWriteableRegistry) { + public RestRollupSearchAction(NamedWriteableRegistry namedWriteableRegistry, Predicate clusterSupportsFeature) { this.namedWriteableRegistry = namedWriteableRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } @Override @@ -51,6 +55,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest, parser, namedWriteableRegistry, + clusterSupportsFeature, size -> searchRequest.source().size(size) ) ); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 4af03808b347a..010c3611c1f96 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -18,6 +18,7 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -38,6 +39,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.FeatureService; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexModule; import org.elasticsearch.indices.SystemIndexDescriptor; @@ -321,6 +323,11 @@ public Collection createComponents(PluginServices services) { Environment environment = services.environment(); ScriptService scriptService = services.scriptService(); NamedXContentRegistry xContentRegistry = services.xContentRegistry(); + FeatureService featureService = services.featureService(); + Predicate clusterSupportsFeature = f -> { + ClusterState state = clusterService.state(); + return state.clusterRecovered() && featureService.clusterHasFeature(state, f); + }; // only initialize these classes if Watcher is enabled, and only after the plugin security policy for Watcher is in place BodyPartSource.init(); @@ -391,7 +398,7 @@ public Collection createComponents(PluginServices services) { ScriptTransform.TYPE, new ScriptTransformFactory(scriptService), SearchTransform.TYPE, - new SearchTransformFactory(settings, client, xContentRegistry, scriptService) + new SearchTransformFactory(settings, client, xContentRegistry, clusterSupportsFeature, scriptService) ) ); @@ -414,7 +421,10 @@ public Collection createComponents(PluginServices services) { // inputs final Map> inputFactories = new HashMap<>(); - inputFactories.put(SearchInput.TYPE, new SearchInputFactory(settings, client, xContentRegistry, scriptService)); + inputFactories.put( + SearchInput.TYPE, + new SearchInputFactory(settings, client, xContentRegistry, clusterSupportsFeature, scriptService) + ); inputFactories.put(SimpleInput.TYPE, new SimpleInputFactory()); inputFactories.put(HttpInput.TYPE, new HttpInputFactory(settings, httpClient, templateEngine)); inputFactories.put(NoneInput.TYPE, new NoneInputFactory()); @@ -504,7 +514,11 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure) final TriggeredWatch.Parser triggeredWatchParser = new TriggeredWatch.Parser(triggerService); final TriggeredWatchStore triggeredWatchStore = new TriggeredWatchStore(settings, client, triggeredWatchParser, bulkProcessor); - final WatcherSearchTemplateService watcherSearchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry); + final WatcherSearchTemplateService watcherSearchTemplateService = new WatcherSearchTemplateService( + scriptService, + xContentRegistry, + clusterSupportsFeature + ); final WatchExecutor watchExecutor = getWatchExecutor(threadPool); final WatchParser watchParser = new WatchParser(triggerService, registry, inputRegistry, cryptoService, getClock()); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java index 9c9ae405bff92..a71fca81360c6 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java @@ -9,6 +9,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; @@ -16,6 +17,7 @@ import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import java.io.IOException; +import java.util.function.Predicate; public class SearchInputFactory extends InputFactory { @@ -23,10 +25,16 @@ public class SearchInputFactory extends InputFactory clusterSupportsFeature, + ScriptService scriptService + ) { this.client = client; this.defaultTimeout = settings.getAsTime("xpack.watcher.input.search.default_timeout", TimeValue.timeValueMinutes(1)); - this.searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry); + this.searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry, clusterSupportsFeature); } @Override diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java index a4c0723455d2d..47f10083ff3a3 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; @@ -24,6 +25,7 @@ import java.io.IOException; import java.util.Map; +import java.util.function.Predicate; /** * {@link WatcherSearchTemplateService} renders {@link WatcherSearchTemplateRequest} before their execution. @@ -32,10 +34,16 @@ public class WatcherSearchTemplateService { private final ScriptService scriptService; private final NamedXContentRegistry xContentRegistry; + private final Predicate clusterSupportsFeature; - public WatcherSearchTemplateService(ScriptService scriptService, NamedXContentRegistry xContentRegistry) { + public WatcherSearchTemplateService( + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Predicate clusterSupportsFeature + ) { this.scriptService = scriptService; this.xContentRegistry = xContentRegistry; + this.clusterSupportsFeature = clusterSupportsFeature; } public String renderTemplate(Script source, WatchExecutionContext ctx, Payload payload) { @@ -73,7 +81,7 @@ public SearchRequest toSearchRequest(WatcherSearchTemplateRequest request) throw XContentHelper.xContentType(source) ) ) { - sourceBuilder.parseXContent(parser, true); + sourceBuilder.parseXContent(parser, true, clusterSupportsFeature); searchRequest.source(sourceBuilder); } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java index 1ef9b9c4a9dbb..8f3c3e57b3c05 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java @@ -10,6 +10,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; @@ -17,6 +18,7 @@ import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import java.io.IOException; +import java.util.function.Predicate; public class SearchTransformFactory extends TransformFactory { @@ -24,11 +26,17 @@ public class SearchTransformFactory extends TransformFactory clusterSupportsFeature, + ScriptService scriptService + ) { super(LogManager.getLogger(ExecutableSearchTransform.class)); this.client = client; this.defaultTimeout = settings.getAsTime("xpack.watcher.transform.search.default_timeout", TimeValue.timeValueMinutes(1)); - this.searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry); + this.searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry, clusterSupportsFeature); } @Override diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java index 172338d60bbe1..2097edd763352 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java @@ -174,7 +174,7 @@ public void testParserValid() throws Exception { XContentParser parser = createParser(builder); parser.nextToken(); - SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, client, xContentRegistry(), scriptService); + SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, client, xContentRegistry(), nf -> false, scriptService); SearchInput searchInput = factory.parseInput("_id", parser); assertEquals(SearchInput.TYPE, searchInput.type()); @@ -211,7 +211,7 @@ public void testThatEmptyRequestBodyWorks() throws Exception { parser.nextToken(); // advance past the first starting object - SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, client, xContentRegistry(), scriptService); + SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, client, xContentRegistry(), nf -> false, scriptService); SearchInput input = factory.parseInput("my-watch", parser); assertThat(input.getRequest(), is(not(nullValue()))); assertThat(input.getRequest().getSearchSource(), is(BytesArray.EMPTY)); @@ -228,6 +228,6 @@ public void testThatEmptyRequestBodyWorks() throws Exception { private WatcherSearchTemplateService watcherSearchTemplateService() { SearchModule module = new SearchModule(Settings.EMPTY, Collections.emptyList()); - return new WatcherSearchTemplateService(scriptService, new NamedXContentRegistry(module.getNamedXContents())); + return new WatcherSearchTemplateService(scriptService, new NamedXContentRegistry(module.getNamedXContents()), nf -> false); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java index ec81a406fb238..9eabdad115f8a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java @@ -71,7 +71,13 @@ public void testParser() throws Exception { ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); Client client = mock(Client.class); - SearchTransformFactory transformFactory = new SearchTransformFactory(Settings.EMPTY, client, xContentRegistry(), scriptService); + SearchTransformFactory transformFactory = new SearchTransformFactory( + Settings.EMPTY, + client, + xContentRegistry(), + nf -> false, + scriptService + ); ExecutableSearchTransform executable = transformFactory.parseExecutable("_id", parser); assertThat(executable, notNullValue()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java index 32347b5d2624b..f2541dc932074 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java @@ -340,7 +340,11 @@ public void testParseWatch_verifyScriptLangDefault() throws Exception { ActionRegistry actionRegistry = registry(Collections.emptyList(), conditionRegistry, transformRegistry); WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, Clock.systemUTC()); - WatcherSearchTemplateService searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry()); + WatcherSearchTemplateService searchTemplateService = new WatcherSearchTemplateService( + scriptService, + xContentRegistry(), + nf -> false + ); XContentBuilder builder = jsonBuilder(); builder.startObject(); @@ -533,7 +537,7 @@ private static ScheduleRegistry registry(Schedule schedule) { private InputRegistry registry(String inputType) { return switch (inputType) { case SearchInput.TYPE -> new InputRegistry( - Map.of(SearchInput.TYPE, new SearchInputFactory(settings, client, xContentRegistry(), scriptService)) + Map.of(SearchInput.TYPE, new SearchInputFactory(settings, client, xContentRegistry(), nf -> false, scriptService)) ); default -> new InputRegistry(Map.of(SimpleInput.TYPE, new SimpleInputFactory())); }; @@ -592,7 +596,7 @@ private TransformRegistry transformRegistry() { ScriptTransform.TYPE, new ScriptTransformFactory(scriptService), SearchTransform.TYPE, - new SearchTransformFactory(settings, client, xContentRegistry(), scriptService) + new SearchTransformFactory(settings, client, xContentRegistry(), nf -> false, scriptService) ) ); } From 05f2c1164e2759d1d76385a2428b65e07945abe6 Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Tue, 13 Feb 2024 11:31:58 -0500 Subject: [PATCH 08/78] [ML] Changed system auditor to use levels (#105429) * Changed system auditor to use levels * Update docs/changelog/105429.yaml --- docs/changelog/105429.yaml | 5 +++++ .../elasticsearch/xpack/ml/notifications/SystemAuditor.java | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/105429.yaml diff --git a/docs/changelog/105429.yaml b/docs/changelog/105429.yaml new file mode 100644 index 0000000000000..706375649b7ca --- /dev/null +++ b/docs/changelog/105429.yaml @@ -0,0 +1,5 @@ +pr: 105429 +summary: Changed system auditor to use levels +area: Machine Learning +type: bug +issues: [] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/SystemAuditor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/SystemAuditor.java index 15c6ab39ef37e..8cfc445e592a0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/SystemAuditor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/SystemAuditor.java @@ -42,12 +42,12 @@ public void info(String resourceId, String message) { @Override public void warning(String resourceId, String message) { assert resourceId == null; - super.info(null, message); + super.warning(null, message); } @Override public void error(String resourceId, String message) { assert resourceId == null; - super.info(null, message); + super.error(null, message); } } From 88f82b5c93542756b512011c8684764cdb02683c Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 13 Feb 2024 16:40:33 +0000 Subject: [PATCH 09/78] [ML] Return chunks for each input to InferenceService::chunkInfer (#105447) --- .../inference/InferenceService.java | 4 +- .../ChunkedSparseEmbeddingResults.java | 4 + .../results/ChunkedTextEmbeddingResults.java | 4 + .../mock/TestInferenceServiceExtension.java | 6 +- .../inference/services/SenderService.java | 4 +- .../services/cohere/CohereService.java | 2 +- .../services/elser/ElserInternalService.java | 34 ++++---- .../huggingface/HuggingFaceBaseService.java | 2 +- .../services/openai/OpenAiService.java | 2 +- .../TextEmbeddingInternalService.java | 34 ++++---- .../services/SenderServiceTests.java | 2 +- .../elser/ElserInternalServiceTests.java | 81 +++++++++++++++++++ .../TextEmbeddingInternalServiceTests.java | 78 ++++++++++++++++++ 13 files changed, 211 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index 605c799a2ba99..a32ffa011fd74 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -103,7 +103,7 @@ void infer( * @param taskSettings Settings in the request to override the model's defaults * @param inputType For search, ingest etc * @param chunkingOptions The window and span options to apply - * @param listener Inference result listener + * @param listener Chunked Inference result listener */ void chunkedInfer( Model model, @@ -111,7 +111,7 @@ void chunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ); /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java index 1ede0386ce314..39b7f6d91acf4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java @@ -37,6 +37,10 @@ public ChunkedSparseEmbeddingResults(StreamInput in) throws IOException { this.chunkedResults = in.readCollectionAsList(ChunkedTextExpansionResults.ChunkedResult::new); } + public List getChunkedResults() { + return chunkedResults; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startArray("sparse_embedding_chunk"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java index 3b3b0e7539bf7..552e78a86d8dc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java @@ -42,6 +42,10 @@ public ChunkedTextEmbeddingResults(StreamInput in) throws IOException { ); } + public List getChunks() { + return chunks; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startArray("text_embedding_chunk"); diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java index fd3c57852028b..215125960c4fc 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java @@ -152,7 +152,7 @@ public void chunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { switch (model.getConfigurations().getTaskType()) { case ANY, SPARSE_EMBEDDING -> listener.onResponse(makeChunkedResults(input)); @@ -177,7 +177,7 @@ private SparseEmbeddingResults makeResults(List input) { return new SparseEmbeddingResults(embeddings); } - private ChunkedSparseEmbeddingResults makeChunkedResults(List input) { + private List makeChunkedResults(List input) { var chunks = new ArrayList(); for (int i = 0; i < input.size(); i++) { var tokens = new ArrayList(); @@ -186,7 +186,7 @@ private ChunkedSparseEmbeddingResults makeChunkedResults(List input) { } chunks.add(new ChunkedTextExpansionResults.ChunkedResult(input.get(i), tokens)); } - return new ChunkedSparseEmbeddingResults(chunks); + return List.of(new ChunkedSparseEmbeddingResults(chunks)); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java index 16e4626dc9ba3..fa329e37fb0e2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java @@ -63,7 +63,7 @@ public void chunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { init(); doChunkedInfer(model, input, taskSettings, inputType, chunkingOptions, listener); @@ -83,7 +83,7 @@ protected abstract void doChunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ); @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 1980ce1f6d9d8..bc511c043fdf3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -181,7 +181,7 @@ protected void doChunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { listener.onFailure(new ElasticsearchStatusException("Chunking not supported by the {} service", RestStatus.BAD_REQUEST, NAME)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index e5adc58c1bb76..8e582f87ceee5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -285,7 +286,7 @@ public void chunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { try { checkCompatibleTaskType(model.getConfigurations().getTaskType()); @@ -355,25 +356,22 @@ private static ElserMlNodeTaskSettings taskSettingsFromMap(TaskType taskType, Ma return ElserMlNodeTaskSettings.DEFAULT; } - private ChunkedSparseEmbeddingResults translateChunkedResults(List inferenceResults) { - if (inferenceResults.size() != 1) { - throw new ElasticsearchStatusException( - "Expected exactly one chunked sparse embedding result", - RestStatus.INTERNAL_SERVER_ERROR - ); - } + private List translateChunkedResults(List inferenceResults) { + var translated = new ArrayList(); - if (inferenceResults.get(0) instanceof ChunkedTextExpansionResults mlChunkedResult) { - return ChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult); - } else { - throw new ElasticsearchStatusException( - "Expected a chunked inference [{}] received [{}]", - RestStatus.INTERNAL_SERVER_ERROR, - ChunkedTextExpansionResults.NAME, - inferenceResults.get(0).getWriteableName() - ); + for (var inferenceResult : inferenceResults) { + if (inferenceResult instanceof ChunkedTextExpansionResults mlChunkedResult) { + translated.add(ChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult)); + } else { + throw new ElasticsearchStatusException( + "Expected a chunked inference [{}] received [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + ChunkedTextExpansionResults.NAME, + inferenceResult.getWriteableName() + ); + } } - + return translated; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java index b592267b7971d..25ba29fddd14e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java @@ -126,7 +126,7 @@ protected void doChunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { listener.onFailure(new UnsupportedOperationException("Chunked inference not implemented for Hugging Face")); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 3b683a4fac240..0f69411b1149b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -190,7 +190,7 @@ protected void doChunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { listener.onFailure(new ElasticsearchStatusException("Chunking not supported by the {} service", RestStatus.BAD_REQUEST, NAME)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java index 3fe8c1d0df694..dc32164be9531 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java @@ -36,13 +36,13 @@ import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; -import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -241,7 +241,7 @@ public void chunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { try { checkCompatibleTaskType(model.getConfigurations().getTaskType()); @@ -345,23 +345,23 @@ private void checkCompatibleTaskType(TaskType taskType) { } } - private ChunkedTextEmbeddingResults translateChunkedResults(List inferenceResults) { - if (inferenceResults.size() != 1) { - throw new ElasticsearchStatusException("Expected exactly one chunked text embedding result", RestStatus.INTERNAL_SERVER_ERROR); - } + private List translateChunkedResults(List inferenceResults) { + var translated = new ArrayList(); - if (inferenceResults.get( - 0 - ) instanceof org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults mlChunkedResult) { - return ChunkedTextEmbeddingResults.ofMlResult(mlChunkedResult); - } else { - throw new ElasticsearchStatusException( - "Expected a chunked inference [{}] received [{}]", - RestStatus.INTERNAL_SERVER_ERROR, - ChunkedTextExpansionResults.NAME, - inferenceResults.get(0).getWriteableName() - ); + for (var inferenceResult : inferenceResults) { + if (inferenceResult instanceof org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults mlChunkedResult) { + translated.add(ChunkedTextEmbeddingResults.ofMlResult(mlChunkedResult)); + } else { + throw new ElasticsearchStatusException( + "Expected a chunked inference [{}] received [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + ChunkedTextEmbeddingResults.NAME, + inferenceResult.getWriteableName() + ); + } } + + return translated; } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java index 4d2d8ee5d7fcf..22a7224d73549 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java @@ -121,7 +121,7 @@ protected void doChunkedInfer( Map taskSettings, InputType inputType, ChunkingOptions chunkingOptions, - ActionListener listener + ActionListener> listener ) { } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java index b098edd5e37bf..6da634afddeb0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java @@ -11,19 +11,37 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceExtension; +import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResultsTests; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ElserInternalServiceTests extends ESTestCase { @@ -307,6 +325,69 @@ public void testParseRequestConfig_DefaultModel() { } } + @SuppressWarnings("unchecked") + public void testChunkInfer() { + var mlTrainedModelResults = new ArrayList(); + mlTrainedModelResults.add(ChunkedTextExpansionResultsTests.createRandomResults()); + mlTrainedModelResults.add(ChunkedTextExpansionResultsTests.createRandomResults()); + var response = new InferTrainedModelDeploymentAction.Response(mlTrainedModelResults); + + ThreadPool threadpool = new TestThreadPool("test"); + Client client = mock(Client.class); + when(client.threadPool()).thenReturn(threadpool); + doAnswer(invocationOnMock -> { + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(response); + return null; + }).when(client) + .execute( + same(InferTrainedModelDeploymentAction.INSTANCE), + any(InferTrainedModelDeploymentAction.Request.class), + any(ActionListener.class) + ); + + var model = new ElserInternalModel( + "foo", + TaskType.SPARSE_EMBEDDING, + "elser", + new ElserInternalServiceSettings(1, 1, "elser"), + new ElserMlNodeTaskSettings() + ); + var service = createService(client); + + var gotResults = new AtomicBoolean(); + var resultsListener = ActionListener.>wrap(chunkedResponse -> { + assertThat(chunkedResponse, hasSize(2)); + assertThat(chunkedResponse.get(0), instanceOf(ChunkedSparseEmbeddingResults.class)); + var result1 = (ChunkedSparseEmbeddingResults) chunkedResponse.get(0); + assertEquals( + ((org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults) mlTrainedModelResults.get(0)).getChunks(), + result1.getChunkedResults() + ); + assertThat(chunkedResponse.get(1), instanceOf(ChunkedSparseEmbeddingResults.class)); + var result2 = (ChunkedSparseEmbeddingResults) chunkedResponse.get(1); + assertEquals( + ((org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(), + result2.getChunkedResults() + ); + gotResults.set(true); + }, ESTestCase::fail); + + service.chunkedInfer( + model, + List.of("foo", "bar"), + Map.of(), + InputType.SEARCH, + new ChunkingOptions(null, null), + ActionListener.runAfter(resultsListener, () -> terminate(threadpool)) + ); + + if (gotResults.get() == false) { + terminate(threadpool); + } + assertTrue("Listener not called", gotResults.get()); + } + private ElserInternalService createService(Client client) { var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client); return new ElserInternalService(context); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalServiceTests.java index d8b808bed3336..42586b0b15f4e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalServiceTests.java @@ -12,22 +12,38 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceExtension; +import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResultsTests; import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TextEmbeddingInternalServiceTests extends ESTestCase { @@ -325,6 +341,68 @@ public void testParsePersistedConfig() { } } + @SuppressWarnings("unchecked") + public void testChunkInfer() { + var mlTrainedModelResults = new ArrayList(); + mlTrainedModelResults.add(ChunkedTextEmbeddingResultsTests.createRandomResults()); + mlTrainedModelResults.add(ChunkedTextEmbeddingResultsTests.createRandomResults()); + var response = new InferTrainedModelDeploymentAction.Response(mlTrainedModelResults); + + ThreadPool threadpool = new TestThreadPool("test"); + Client client = mock(Client.class); + when(client.threadPool()).thenReturn(threadpool); + doAnswer(invocationOnMock -> { + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(response); + return null; + }).when(client) + .execute( + same(InferTrainedModelDeploymentAction.INSTANCE), + any(InferTrainedModelDeploymentAction.Request.class), + any(ActionListener.class) + ); + + var model = new MultilingualE5SmallModel( + "foo", + TaskType.TEXT_EMBEDDING, + "e5", + new MultilingualE5SmallInternalServiceSettings(1, 1, "cross-platform") + ); + var service = createService(client); + + var gotResults = new AtomicBoolean(); + var resultsListener = ActionListener.>wrap(chunkedResponse -> { + assertThat(chunkedResponse, hasSize(2)); + assertThat(chunkedResponse.get(0), instanceOf(ChunkedTextEmbeddingResults.class)); + var result1 = (ChunkedTextEmbeddingResults) chunkedResponse.get(0); + assertEquals( + ((org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults) mlTrainedModelResults.get(0)).getChunks(), + result1.getChunks() + ); + assertThat(chunkedResponse.get(1), instanceOf(ChunkedTextEmbeddingResults.class)); + var result2 = (ChunkedTextEmbeddingResults) chunkedResponse.get(1); + assertEquals( + ((org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults) mlTrainedModelResults.get(1)).getChunks(), + result2.getChunks() + ); + gotResults.set(true); + }, ESTestCase::fail); + + service.chunkedInfer( + model, + List.of("foo", "bar"), + Map.of(), + InputType.SEARCH, + new ChunkingOptions(null, null), + ActionListener.runAfter(resultsListener, () -> terminate(threadpool)) + ); + + if (gotResults.get() == false) { + terminate(threadpool); + } + assertTrue("Listener not called", gotResults.get()); + } + private TextEmbeddingInternalService createService(Client client) { var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client); return new TextEmbeddingInternalService(context); From 89bf949555f4a75b02486915a775892d0ab407d0 Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Tue, 13 Feb 2024 12:31:07 -0500 Subject: [PATCH 10/78] [ML] Fix for inference modelId trained model deployment id collision (#105303) * Fix for inference modelId trained model deployment id collision * Add check for model already downloaded before put trained model --- .../inference/InferenceService.java | 11 ++++++ .../core/ml/action/PutTrainedModelAction.java | 2 + .../xpack/core/ml/utils/ExceptionsHelper.java | 3 +- .../inference/InferenceBaseRestTest.java | 25 ++++++++++++ .../xpack/inference/TextEmbeddingCrudIT.java | 35 ++++++++--------- .../TransportPutInferenceModelAction.java | 38 +++++++++++-------- .../services/elser/ElserInternalService.java | 26 +++++++++++++ .../TextEmbeddingInternalService.java | 36 +++++++++++++++++- .../TransportPutTrainedModelAction.java | 3 +- .../test/ml/3rd_party_deployment.yml | 2 +- 10 files changed, 141 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index a32ffa011fd74..ccf405e1074e6 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -142,6 +142,17 @@ default void putModel(Model modelVariant, ActionListener listener) { listener.onResponse(true); } + /** + * Checks if the modelId has been downloaded to the local Elasticsearch cluster using the trained models API + * The default action does nothing except acknowledge the request (false). + * Any internal services should Override this method. + * @param model + * @param listener The listener + */ + default void isModelDownloaded(Model model, ActionListener listener) { + listener.onResponse(false); + }; + /** * Optionally test the new model configuration in the inference service. * This function should be called when the model is first created, the diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAction.java index 769a3a3dded28..2e5a475369510 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAction.java @@ -30,6 +30,8 @@ public class PutTrainedModelAction extends ActionType putModelInternal(String endpoint, String modelConfig return entityAsMap(response); } + protected Map putE5TrainedModels() throws IOException { + var request = new Request("PUT", "_ml/trained_models/.multilingual-e5-small?wait_for_completion=true"); + + String body = """ + { + "input": { + "field_names": ["text_field"] + } + } + """; + + request.setJsonEntity(body); + var response = client().performRequest(request); + assertOkOrCreated(response); + return entityAsMap(response); + } + + protected Map deployE5TrainedModels() throws IOException { + var request = new Request("POST", "_ml/trained_models/.multilingual-e5-small/deployment/_start?wait_for=fully_allocated"); + + var response = client().performRequest(request); + assertOkOrCreated(response); + return entityAsMap(response); + } + protected Map getModel(String modelId) throws IOException { var endpoint = Strings.format("_inference/%s", modelId); return getAllModelInternal(endpoint); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java index 24a701095ecb7..7fb47e901f703 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java @@ -24,29 +24,18 @@ public class TextEmbeddingCrudIT extends InferenceBaseRestTest { public void testPutE5Small_withNoModelVariant() throws IOException { - // Model downloaded automatically & test infer with no model variant { String inferenceEntityId = randomAlphaOfLength(10).toLowerCase(); - putTextEmbeddingModel(inferenceEntityId, TaskType.TEXT_EMBEDDING, noModelIdVariantJsonEntity()); - var models = getTrainedModel("_all"); - assertThat(models.toString(), containsString("deployment_id=" + inferenceEntityId)); - - Map results = inferOnMockService( - inferenceEntityId, - TaskType.TEXT_EMBEDDING, - List.of("hello world", "this is the second document") + expectThrows( + org.elasticsearch.client.ResponseException.class, + () -> putTextEmbeddingModel(inferenceEntityId, noModelIdVariantJsonEntity()) ); - assertTrue(((List) ((Map) ((List) results.get("text_embedding")).get(0)).get("embedding")).size() > 1); - // there exists embeddings - assertTrue(((List) results.get("text_embedding")).size() == 2); - // there are two sets of embeddings - deleteTextEmbeddingModel(inferenceEntityId); } } public void testPutE5Small_withPlatformAgnosticVariant() throws IOException { String inferenceEntityId = randomAlphaOfLength(10).toLowerCase(); - putTextEmbeddingModel(inferenceEntityId, TaskType.TEXT_EMBEDDING, platformAgnosticModelVariantJsonEntity()); + putTextEmbeddingModel(inferenceEntityId, platformAgnosticModelVariantJsonEntity()); var models = getTrainedModel("_all"); assertThat(models.toString(), containsString("deployment_id=" + inferenceEntityId)); @@ -65,7 +54,7 @@ public void testPutE5Small_withPlatformAgnosticVariant() throws IOException { public void testPutE5Small_withPlatformSpecificVariant() throws IOException { String inferenceEntityId = randomAlphaOfLength(10).toLowerCase(); if ("linux-x86_64".equals(Platforms.PLATFORM_NAME)) { - putTextEmbeddingModel(inferenceEntityId, TaskType.TEXT_EMBEDDING, platformSpecificModelVariantJsonEntity()); + putTextEmbeddingModel(inferenceEntityId, platformSpecificModelVariantJsonEntity()); var models = getTrainedModel("_all"); assertThat(models.toString(), containsString("deployment_id=" + inferenceEntityId)); @@ -82,7 +71,7 @@ public void testPutE5Small_withPlatformSpecificVariant() throws IOException { } else { expectThrows( org.elasticsearch.client.ResponseException.class, - () -> putTextEmbeddingModel(inferenceEntityId, TaskType.TEXT_EMBEDDING, platformSpecificModelVariantJsonEntity()) + () -> putTextEmbeddingModel(inferenceEntityId, platformSpecificModelVariantJsonEntity()) ); } } @@ -91,9 +80,15 @@ public void testPutE5Small_withFakeModelVariant() { String inferenceEntityId = randomAlphaOfLength(10).toLowerCase(); expectThrows( org.elasticsearch.client.ResponseException.class, - () -> putTextEmbeddingModel(inferenceEntityId, TaskType.TEXT_EMBEDDING, fakeModelVariantJsonEntity()) + () -> putTextEmbeddingModel(inferenceEntityId, fakeModelVariantJsonEntity()) ); + } + public void testPutE5WithTrainedModelAndInference() throws IOException { + putE5TrainedModels(); + deployE5TrainedModels(); + putTextEmbeddingModel("an-e5-deployment", platformAgnosticModelVariantJsonEntity()); + getTrainedModel("an-e5-deployment"); } private Map deleteTextEmbeddingModel(String inferenceEntityId) throws IOException { @@ -104,8 +99,8 @@ private Map deleteTextEmbeddingModel(String inferenceEntityId) t return entityAsMap(response); } - private Map putTextEmbeddingModel(String inferenceEntityId, TaskType taskType, String jsonEntity) throws IOException { - var endpoint = Strings.format("_inference/%s/%s", taskType, inferenceEntityId); + private Map putTextEmbeddingModel(String inferenceEntityId, String jsonEntity) throws IOException { + var endpoint = Strings.format("_inference/%s/%s", TaskType.TEXT_EMBEDDING, inferenceEntityId); var request = new Request("PUT", endpoint); request.setJsonEntity(jsonEntity); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java index b31703f44f6f9..6667e314a62b8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java @@ -210,23 +210,31 @@ private void parseAndStoreModel( } private void putAndStartModel(InferenceService service, Model model, ActionListener finalListener) { - SubscribableListener.newForked((listener1) -> { service.putModel(model, listener1); }).< - PutInferenceModelAction.Response>andThen((listener2, modelDidPut) -> { - if (modelDidPut) { - if (skipValidationAndStart) { - listener2.onResponse(new PutInferenceModelAction.Response(model.getConfigurations())); - } else { - service.start( - model, - listener2.delegateFailureAndWrap( - (l3, ok) -> l3.onResponse(new PutInferenceModelAction.Response(model.getConfigurations())) - ) - ); - } + SubscribableListener.newForked(listener -> { + var errorCatchingListener = ActionListener.wrap(listener::onResponse, e -> { listener.onResponse(false); }); + service.isModelDownloaded(model, errorCatchingListener); + }).andThen((listener, isDownloaded) -> { + if (isDownloaded == false) { + service.putModel(model, listener); + } else { + listener.onResponse(true); + } + }).andThen((listener, modelDidPut) -> { + if (modelDidPut) { + if (skipValidationAndStart) { + listener.onResponse(new PutInferenceModelAction.Response(model.getConfigurations())); } else { - logger.warn("Failed to put model [{}]", model.getInferenceEntityId()); + service.start( + model, + listener.delegateFailureAndWrap( + (l3, ok) -> l3.onResponse(new PutInferenceModelAction.Response(model.getConfigurations())) + ) + ); } - }).addListener(finalListener); + } else { + logger.warn("Failed to put model [{}]", model.getInferenceEntityId()); + } + }).addListener(finalListener); } private Map requestToMap(PutInferenceModelAction.Request request) throws IOException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index 8e582f87ceee5..8e4918a37b5a4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; +import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; @@ -347,6 +348,31 @@ public void putModel(Model model, ActionListener listener) { } } + @Override + public void isModelDownloaded(Model model, ActionListener listener) { + ActionListener getModelsResponseListener = listener.delegateFailure((delegate, response) -> { + if (response.getResources().count() < 1) { + delegate.onResponse(Boolean.FALSE); + } else { + delegate.onResponse(Boolean.TRUE); + } + }); + + if (model instanceof ElserInternalModel elserModel) { + String modelId = elserModel.getServiceSettings().getModelId(); + GetTrainedModelsAction.Request getRequest = new GetTrainedModelsAction.Request(modelId); + executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener); + } else { + listener.onFailure( + new IllegalArgumentException( + "Can not download model automatically for [" + + model.getConfigurations().getInferenceEntityId() + + "] you may need to download it through the trained models API or with eland." + ) + ); + } + } + private static ElserMlNodeTaskSettings taskSettingsFromMap(TaskType taskType, Map config) { if (taskType != TaskType.SPARSE_EMBEDDING) { throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java index dc32164be9531..59228c6dcbddf 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/textembedding/TextEmbeddingInternalService.java @@ -314,8 +314,13 @@ public void putModel(Model model, ActionListener listener) { INFERENCE_ORIGIN, PutTrainedModelAction.INSTANCE, putRequest, - listener.delegateFailure((l, r) -> { - l.onResponse(Boolean.TRUE); + ActionListener.wrap(response -> listener.onResponse(Boolean.TRUE), e -> { + if (e instanceof ElasticsearchStatusException esException + && esException.getMessage().contains(PutTrainedModelAction.MODEL_ALREADY_EXISTS_ERROR_MESSAGE_FRAGMENT)) { + listener.onResponse(Boolean.TRUE); + } else { + listener.onFailure(e); + } }) ); } else if (model instanceof CustomElandModel elandModel) { @@ -333,6 +338,33 @@ public void putModel(Model model, ActionListener listener) { } } + @Override + public void isModelDownloaded(Model model, ActionListener listener) { + ActionListener getModelsResponseListener = listener.delegateFailure((delegate, response) -> { + if (response.getResources().count() < 1) { + delegate.onResponse(Boolean.FALSE); + } else { + delegate.onResponse(Boolean.TRUE); + } + }); + + if (model instanceof TextEmbeddingModel == false) { + listener.onFailure(notTextEmbeddingModelException(model)); + } else if (model.getServiceSettings() instanceof InternalServiceSettings internalServiceSettings) { + String modelId = internalServiceSettings.getModelId(); + GetTrainedModelsAction.Request getRequest = new GetTrainedModelsAction.Request(modelId); + executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener); + } else { + listener.onFailure( + new IllegalArgumentException( + "Unable to determine supported model for [" + + model.getConfigurations().getInferenceEntityId() + + "] please verify the request and submit a bug report if necessary." + ) + ); + } + } + private static IllegalStateException notTextEmbeddingModelException(Model model) { return new IllegalStateException( "Error starting model, [" + model.getConfigurations().getInferenceEntityId() + "] is not a text embedding model" diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java index edbb9f297c8cd..a272aea59cbc4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java @@ -81,6 +81,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction.MODEL_ALREADY_EXISTS_ERROR_MESSAGE_FRAGMENT; public class TransportPutTrainedModelAction extends TransportMasterNodeAction { @@ -230,7 +231,7 @@ protected void masterOperation( if (TrainedModelAssignmentMetadata.fromState(state).hasDeployment(trainedModelConfig.getModelId())) { finalResponseListener.onFailure( ExceptionsHelper.badRequestException( - "Cannot create model [{}] the id is the same as an current model deployment", + "Cannot create model [{}] " + MODEL_ALREADY_EXISTS_ERROR_MESSAGE_FRAGMENT, config.getModelId() ) ); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml index fdccf473b358a..e8ac7ce3694e8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/3rd_party_deployment.yml @@ -688,7 +688,7 @@ setup: - match: { assignment.assignment_state: started } - do: - catch: /Cannot create model \[test_model_deployment\] the id is the same as an current model deployment/ + catch: /Cannot create model \[test_model_deployment\] the model id is the same as the deployment id of a current model deployment/ ml.put_trained_model: model_id: test_model_deployment body: > From 05d2375e61e161d267d1e3fc8145a083168ed476 Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Tue, 13 Feb 2024 13:25:34 -0500 Subject: [PATCH 11/78] [ML] System auditor notifications for pytorch restarts (#105411) * Add systemAudit logs to process restarts * Updated error message for system audit / notifications * Added system audit message for not restarting pytorch process * Switch to inferenceAuditor and update error message --- .../xpack/ml/MachineLearning.java | 9 +++- .../deployment/DeploymentManager.java | 54 ++++++++++++++----- .../deployment/DeploymentManagerTests.java | 6 ++- 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 6916a04084285..fb908bd79bab1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1151,7 +1151,14 @@ public Collection createComponents(PluginServices services) { ); this.deploymentManager.set( - new DeploymentManager(client, xContentRegistry, threadPool, pyTorchProcessFactory, getMaxModelDeploymentsPerNode()) + new DeploymentManager( + client, + xContentRegistry, + threadPool, + pyTorchProcessFactory, + getMaxModelDeploymentsPerNode(), + inferenceAuditor + ) ); // Data frame analytics components diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java index d9d6adbba4737..17b931d971188 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java @@ -53,6 +53,7 @@ import org.elasticsearch.xpack.ml.inference.pytorch.process.PyTorchResultProcessor; import org.elasticsearch.xpack.ml.inference.pytorch.process.PyTorchStateStreamer; import org.elasticsearch.xpack.ml.inference.pytorch.results.ThreadSettings; +import org.elasticsearch.xpack.ml.notifications.InferenceAuditor; import java.io.IOException; import java.time.Duration; @@ -85,6 +86,7 @@ public class DeploymentManager { private final ExecutorService executorServiceForDeployment; private final ExecutorService executorServiceForProcess; private final ThreadPool threadPool; + private final InferenceAuditor inferenceAuditor; private final ConcurrentMap processContextByAllocation = new ConcurrentHashMap<>(); private final int maxProcesses; @@ -93,12 +95,14 @@ public DeploymentManager( NamedXContentRegistry xContentRegistry, ThreadPool threadPool, PyTorchProcessFactory pyTorchProcessFactory, - int maxProcesses + int maxProcesses, + InferenceAuditor inferenceAuditor ) { this.client = Objects.requireNonNull(client); this.xContentRegistry = Objects.requireNonNull(xContentRegistry); this.pyTorchProcessFactory = Objects.requireNonNull(pyTorchProcessFactory); this.threadPool = Objects.requireNonNull(threadPool); + this.inferenceAuditor = Objects.requireNonNull(inferenceAuditor); this.executorServiceForDeployment = threadPool.executor(UTILITY_THREAD_POOL_NAME); this.executorServiceForProcess = threadPool.executor(MachineLearning.NATIVE_INFERENCE_COMMS_THREAD_POOL_NAME); this.maxProcesses = maxProcesses; @@ -523,7 +527,7 @@ synchronized void startAndLoad(TrainedModelLocation modelLocation, ActionListene task, executorServiceForProcess, () -> resultProcessor.awaitCompletion(COMPLETION_TIMEOUT.getMinutes(), TimeUnit.MINUTES), - onProcessCrashHandleRestarts(startsCount) + onProcessCrashHandleRestarts(startsCount, task.getDeploymentId()) ) ); startTime = Instant.now(); @@ -546,16 +550,19 @@ synchronized void startAndLoad(TrainedModelLocation modelLocation, ActionListene } } - private Consumer onProcessCrashHandleRestarts(AtomicInteger startsCount) { + private Consumer onProcessCrashHandleRestarts(AtomicInteger startsCount, String deploymentId) { return (reason) -> { if (isThisProcessOlderThan1Day()) { startsCount.set(1); - logger.error( - "[{}] inference process crashed due to reason [{}]. This process was started more than 24 hours ago; " - + "the starts count is reset to 1.", - task.getDeploymentId(), - reason - ); + { + String logMessage = "[" + + task.getDeploymentId() + + "] inference process crashed due to reason [" + + reason + + "]. This process was started more than 24 hours ago; " + + "the starts count is reset to 1."; + logger.error(logMessage); + } } else { logger.error("[{}] inference process crashed due to reason [{}]", task.getDeploymentId(), reason); } @@ -566,20 +573,32 @@ private Consumer onProcessCrashHandleRestarts(AtomicInteger startsCount) stateStreamer.cancel(); if (startsCount.get() <= NUM_RESTART_ATTEMPTS) { - logger.info("[{}] restarting inference process after [{}] starts", task.getDeploymentId(), startsCount.get()); + { + String logAndAuditMessage = "Inference process [" + + task.getDeploymentId() + + "] failed due to [" + + reason + + "]. This is the [" + + startsCount.get() + + "] failure in 24 hours, and the process will be restarted."; + logger.info(logAndAuditMessage); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute(() -> inferenceAuditor.warning(deploymentId, logAndAuditMessage)); + } priorityProcessWorker.shutdownNow(); // TODO what to do with these tasks? ActionListener errorListener = ActionListener.wrap((trainedModelDeploymentTask -> { logger.debug("Completed restart of inference process, the [{}] start", startsCount); }), (e) -> finishClosingProcess( startsCount, - "Failed to restart inference process because of error [" + e.getMessage() + "]" + "Failed to restart inference process because of error [" + e.getMessage() + "]", + deploymentId ) ); startDeployment(task, startsCount.incrementAndGet(), errorListener); } else { - finishClosingProcess(startsCount, reason); + finishClosingProcess(startsCount, reason, deploymentId); } }; } @@ -588,8 +607,15 @@ private boolean isThisProcessOlderThan1Day() { return startTime.isBefore(Instant.now().minus(Duration.ofDays(1))); } - private void finishClosingProcess(AtomicInteger startsCount, String reason) { - logger.warn("[{}] inference process failed after [{}] starts, not restarting again", task.getDeploymentId(), startsCount.get()); + private void finishClosingProcess(AtomicInteger startsCount, String reason, String deploymentId) { + String logAndAuditMessage = "[" + + task.getDeploymentId() + + "] inference process failed after [" + + startsCount.get() + + "] starts in 24 hours, not restarting again."; + logger.warn(logAndAuditMessage); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute(() -> inferenceAuditor.error(deploymentId, logAndAuditMessage)); priorityProcessWorker.shutdownNowWithError(new IllegalStateException(reason)); if (nlpTaskProcessor.get() != null) { nlpTaskProcessor.get().close(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java index ffa56d3d076f9..4406760998ef0 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.ml.inference.pytorch.PriorityProcessWorkerExecutorService; import org.elasticsearch.xpack.ml.inference.pytorch.process.PyTorchProcessFactory; import org.elasticsearch.xpack.ml.inference.pytorch.process.PyTorchResultProcessor; +import org.elasticsearch.xpack.ml.notifications.InferenceAuditor; import org.junit.After; import org.junit.Before; @@ -36,6 +37,7 @@ public class DeploymentManagerTests extends ESTestCase { private ThreadPool tp; + private InferenceAuditor inferenceAuditor; @Before public void managerSetup() { @@ -58,6 +60,7 @@ public void managerSetup() { "xpack.ml.native_inference_comms_thread_pool" ) ); + inferenceAuditor = mock(InferenceAuditor.class); } @After @@ -78,7 +81,8 @@ public void testRejectedExecution() { mock(NamedXContentRegistry.class), tp, mock(PyTorchProcessFactory.class), - 10 + 10, + inferenceAuditor ); PriorityProcessWorkerExecutorService priorityExecutorService = new PriorityProcessWorkerExecutorService( From ecb01405e7a0aa21513d68e2c0ad0270e9d1a687 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 13 Feb 2024 18:44:33 +0000 Subject: [PATCH 12/78] [ML] Move the model parameter from task settings to service settings (#105458) When configuring an OpenAI text embedding service the `model_id` should have always been part of the service settings rather than task settings. Task settings are overridable, service settings cannot be changed. If different models are used the configured entities are considered distinct. task_settings is now optional as it contains a single optional field (`user`) ``` PUT _inference/text_embedding/openai_embeddings { "service": "openai", "service_settings": { "api_key": "XXX", "model_id": "text-embedding-ada-002" } } ``` Backwards compatibility with previously configured models is maintained by moving the `model_id` (or `model`) from task settings to service settings at the first stage of parsing. New configurations are persisted with `model_id` in service settings, old configurations with `model_id` in task settings are not modified and will be tolerated by a lenient parser. --- docs/changelog/105458.yaml | 5 + .../org/elasticsearch/TransportVersions.java | 1 + .../openai/OpenAiEmbeddingsRequest.java | 2 +- .../inference/services/ServiceFields.java | 1 + .../inference/services/ServiceUtils.java | 10 + .../services/openai/OpenAiService.java | 46 ++++- .../OpenAiEmbeddingsRequestTaskSettings.java | 20 +- .../OpenAiEmbeddingsServiceSettings.java | 40 +++- .../OpenAiEmbeddingsTaskSettings.java | 84 ++++----- .../openai/OpenAiActionCreatorTests.java | 8 +- .../services/openai/OpenAiServiceTests.java | 172 ++++++++++++------ .../OpenAiEmbeddingsModelTests.java | 18 +- ...nAiEmbeddingsRequestTaskSettingsTests.java | 52 +----- .../OpenAiEmbeddingsServiceSettingsTests.java | 67 +++++-- .../OpenAiEmbeddingsTaskSettingsTests.java | 164 +++-------------- 15 files changed, 336 insertions(+), 354 deletions(-) create mode 100644 docs/changelog/105458.yaml diff --git a/docs/changelog/105458.yaml b/docs/changelog/105458.yaml new file mode 100644 index 0000000000000..2bab415884975 --- /dev/null +++ b/docs/changelog/105458.yaml @@ -0,0 +1,5 @@ +pr: 105458 +summary: The OpenAI model parameter should be in service settings not task settings. Move the configuration field to service settings +area: Machine Learning +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 668d0ad32958c..6ac20c1753664 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -132,6 +132,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_DIMENSIONS_SET_BY_USER_ADDED = def(8_592_00_0); public static final TransportVersion INDEX_REQUEST_NORMALIZED_BYTES_PARSED = def(8_593_00_0); public static final TransportVersion INGEST_GRAPH_STRUCTURE_EXCEPTION = def(8_594_00_0); + public static final TransportVersion ML_MODEL_IN_SERVICE_SETTINGS = def(8_595_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java index 8209a10974151..9893b556e1a47 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java @@ -56,7 +56,7 @@ public HttpRequest createHttpRequest() { Strings.toString( new OpenAiEmbeddingsRequestEntity( truncationResult.input(), - model.getTaskSettings().modelId(), + model.getServiceSettings().modelId(), model.getTaskSettings().user(), model.getServiceSettings().dimensions(), model.getServiceSettings().dimensionsSetByUser() diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java index 80e6e4a6124ec..8d83a8a81ec0d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java @@ -16,6 +16,7 @@ public final class ServiceFields { public static final String DIMENSIONS = "dimensions"; public static final String MAX_INPUT_TOKENS = "max_input_tokens"; public static final String URL = "url"; + public static final String MODEL_ID = "model_id"; private ServiceFields() { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 1331d13ba60e4..532fd2359ac2b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -26,6 +26,7 @@ import java.net.URISyntaxException; import java.util.Arrays; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -75,6 +76,15 @@ public static Map removeFromMapOrThrowIfNull(Map return value; } + @SuppressWarnings("unchecked") + public static Map removeFromMapOrDefaultEmpty(Map sourceMap, String fieldName) { + Map value = (Map) sourceMap.remove(fieldName); + if (value == null) { + return new HashMap<>(); + } + return value; + } + public static String removeStringOrThrowIfNull(Map sourceMap, String key) { String value = removeAsType(sourceMap, key, String.class); if (value == null) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 0f69411b1149b..7010b59990cd3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -36,8 +36,10 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; @@ -63,7 +65,9 @@ public void parseRequestConfig( ) { try { Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); - Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + moveModelFromTaskToServiceSettings(taskSettingsMap, serviceSettingsMap); OpenAiModel model = createModel( inferenceEntityId, @@ -85,7 +89,7 @@ public void parseRequestConfig( } } - private static OpenAiModel createModelWithoutLoggingDeprecations( + private static OpenAiModel createModelFromPersistent( String inferenceEntityId, TaskType taskType, Map serviceSettings, @@ -136,9 +140,11 @@ public OpenAiModel parsePersistedConfigWithSecrets( ) { Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); - Map secretSettingsMap = removeFromMapOrThrowIfNull(secrets, ModelSecrets.SECRET_SETTINGS); + Map secretSettingsMap = removeFromMapOrDefaultEmpty(secrets, ModelSecrets.SECRET_SETTINGS); + + moveModelFromTaskToServiceSettings(taskSettingsMap, serviceSettingsMap); - return createModelWithoutLoggingDeprecations( + return createModelFromPersistent( inferenceEntityId, taskType, serviceSettingsMap, @@ -151,9 +157,11 @@ public OpenAiModel parsePersistedConfigWithSecrets( @Override public OpenAiModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map config) { Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); - Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + moveModelFromTaskToServiceSettings(taskSettingsMap, serviceSettingsMap); - return createModelWithoutLoggingDeprecations( + return createModelFromPersistent( inferenceEntityId, taskType, serviceSettingsMap, @@ -232,6 +240,7 @@ private OpenAiEmbeddingsModel updateModelWithEmbeddingDetails(OpenAiEmbeddingsMo } OpenAiEmbeddingsServiceSettings serviceSettings = new OpenAiEmbeddingsServiceSettings( + model.getServiceSettings().modelId(), model.getServiceSettings().uri(), model.getServiceSettings().organizationId(), SimilarityMeasure.DOT_PRODUCT, @@ -247,4 +256,29 @@ private OpenAiEmbeddingsModel updateModelWithEmbeddingDetails(OpenAiEmbeddingsMo public TransportVersion getMinimalSupportedVersion() { return TransportVersions.V_8_12_0; } + + /** + * Model was originally defined in task settings, but it should + * have been part of the service settings. + * + * If model or model_id are in the task settings map move + * them to service settings ready for parsing + * + * @param taskSettings Task settings map + * @param serviceSettings Service settings map + */ + static void moveModelFromTaskToServiceSettings(Map taskSettings, Map serviceSettings) { + if (serviceSettings.containsKey(MODEL_ID)) { + return; + } + + final String OLD_MODEL_ID_FIELD = "model"; + var oldModelId = taskSettings.remove(OLD_MODEL_ID_FIELD); + if (oldModelId != null) { + serviceSettings.put(MODEL_ID, oldModelId); + } else { + var modelId = taskSettings.remove(MODEL_ID); + serviceSettings.put(MODEL_ID, modelId); + } + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettings.java index 221bd61214455..5bdb0d7542a83 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettings.java @@ -16,25 +16,23 @@ import java.util.Map; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString; -import static org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsTaskSettings.MODEL_ID; -import static org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD; import static org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsTaskSettings.USER; /** * This class handles extracting OpenAI task settings from a request. The difference between this class and * {@link OpenAiEmbeddingsTaskSettings} is that this class considers all fields as optional. It will not throw an error if a field * is missing. This allows overriding persistent task settings. - * @param modelId the name of the model to use with this request * @param user a unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse */ -public record OpenAiEmbeddingsRequestTaskSettings(@Nullable String modelId, @Nullable String user) { +public record OpenAiEmbeddingsRequestTaskSettings(@Nullable String user) { private static final Logger logger = LogManager.getLogger(OpenAiEmbeddingsRequestTaskSettings.class); - public static final OpenAiEmbeddingsRequestTaskSettings EMPTY_SETTINGS = new OpenAiEmbeddingsRequestTaskSettings(null, null); + public static final OpenAiEmbeddingsRequestTaskSettings EMPTY_SETTINGS = new OpenAiEmbeddingsRequestTaskSettings(null); /** * Extracts the task settings from a map. All settings are considered optional and the absence of a setting * does not throw an error. + * * @param map the settings received from a request * @return a {@link OpenAiEmbeddingsRequestTaskSettings} */ @@ -45,22 +43,12 @@ public static OpenAiEmbeddingsRequestTaskSettings fromMap(Map ma ValidationException validationException = new ValidationException(); - // I'm intentionally not logging if this is set because it would log on every request - String model = extractOptionalString(map, OLD_MODEL_ID_FIELD, ModelConfigurations.TASK_SETTINGS, validationException); - - String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.TASK_SETTINGS, validationException); String user = extractOptionalString(map, USER, ModelConfigurations.TASK_SETTINGS, validationException); - var modelIdToUse = getModelId(model, modelId); - if (validationException.validationErrors().isEmpty() == false) { throw validationException; } - return new OpenAiEmbeddingsRequestTaskSettings(modelIdToUse, user); - } - - private static String getModelId(@Nullable String model, @Nullable String modelId) { - return modelId != null ? modelId : model; + return new OpenAiEmbeddingsRequestTaskSettings(user); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java index 4f72321d450fb..229e45a024458 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java @@ -28,11 +28,13 @@ import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY; import static org.elasticsearch.xpack.inference.services.ServiceFields.URL; import static org.elasticsearch.xpack.inference.services.ServiceUtils.convertToUri; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createOptionalUri; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType; @@ -92,11 +94,13 @@ private static CommonFields fromMap(Map map, ValidationException Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class); Integer dims = removeAsType(map, DIMENSIONS, Integer.class); URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException); + String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); - return new CommonFields(uri, organizationId, similarity, maxInputTokens, dims); + return new CommonFields(modelId, uri, organizationId, similarity, maxInputTokens, dims); } private record CommonFields( + String modelId, @Nullable URI uri, @Nullable String organizationId, @Nullable SimilarityMeasure similarity, @@ -104,6 +108,7 @@ private record CommonFields( @Nullable Integer dimensions ) {} + private final String modelId; private final URI uri; private final String organizationId; private final SimilarityMeasure similarity; @@ -112,6 +117,7 @@ private record CommonFields( private final Boolean dimensionsSetByUser; public OpenAiEmbeddingsServiceSettings( + String modelId, @Nullable URI uri, @Nullable String organizationId, @Nullable SimilarityMeasure similarity, @@ -120,6 +126,7 @@ public OpenAiEmbeddingsServiceSettings( Boolean dimensionsSetByUser ) { this.uri = uri; + this.modelId = modelId; this.organizationId = organizationId; this.similarity = similarity; this.dimensions = dimensions; @@ -127,7 +134,8 @@ public OpenAiEmbeddingsServiceSettings( this.dimensionsSetByUser = Objects.requireNonNull(dimensionsSetByUser); } - public OpenAiEmbeddingsServiceSettings( + OpenAiEmbeddingsServiceSettings( + String modelId, @Nullable String uri, @Nullable String organizationId, @Nullable SimilarityMeasure similarity, @@ -135,7 +143,7 @@ public OpenAiEmbeddingsServiceSettings( @Nullable Integer maxInputTokens, Boolean dimensionsSetByUser ) { - this(createOptionalUri(uri), organizationId, similarity, dimensions, maxInputTokens, dimensionsSetByUser); + this(modelId, createOptionalUri(uri), organizationId, similarity, dimensions, maxInputTokens, dimensionsSetByUser); } public OpenAiEmbeddingsServiceSettings(StreamInput in) throws IOException { @@ -156,10 +164,23 @@ public OpenAiEmbeddingsServiceSettings(StreamInput in) throws IOException { } else { dimensionsSetByUser = false; } + if (in.getTransportVersion().onOrAfter(TransportVersions.ML_MODEL_IN_SERVICE_SETTINGS)) { + modelId = in.readString(); + } else { + modelId = "unset"; + } } private OpenAiEmbeddingsServiceSettings(CommonFields fields, Boolean dimensionsSetByUser) { - this(fields.uri, fields.organizationId, fields.similarity, fields.dimensions, fields.maxInputTokens, dimensionsSetByUser); + this( + fields.modelId, + fields.uri, + fields.organizationId, + fields.similarity, + fields.dimensions, + fields.maxInputTokens, + dimensionsSetByUser + ); } public URI uri() { @@ -186,6 +207,10 @@ public Integer maxInputTokens() { return maxInputTokens; } + public String modelId() { + return modelId; + } + @Override public String getWriteableName() { return NAME; @@ -206,6 +231,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private void toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + builder.field(MODEL_ID, modelId); if (uri != null) { builder.field(URL, uri.toString()); } @@ -254,6 +280,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.ML_DIMENSIONS_SET_BY_USER_ADDED)) { out.writeBoolean(dimensionsSetByUser); } + if (out.getTransportVersion().onOrAfter(TransportVersions.ML_MODEL_IN_SERVICE_SETTINGS)) { + out.writeString(modelId); + } } @Override @@ -262,6 +291,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; OpenAiEmbeddingsServiceSettings that = (OpenAiEmbeddingsServiceSettings) o; return Objects.equals(uri, that.uri) + && Objects.equals(modelId, that.modelId) && Objects.equals(organizationId, that.organizationId) && Objects.equals(similarity, that.similarity) && Objects.equals(dimensions, that.dimensions) @@ -271,6 +301,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(uri, organizationId, similarity, dimensions, maxInputTokens, dimensionsSetByUser); + return Objects.hash(uri, modelId, organizationId, similarity, dimensions, maxInputTokens, dimensionsSetByUser); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java index fec49e962734a..1a202e8ca8249 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java @@ -7,8 +7,6 @@ package org.elasticsearch.xpack.inference.services.openai.embeddings; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; @@ -18,7 +16,6 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.openai.OpenAiParseContext; import java.io.IOException; @@ -30,53 +27,23 @@ /** * Defines the task settings for the openai service. * - * @param modelId the id of the model to use in the requests to openai - * @param user an optional unique identifier representing the end-user, which can help OpenAI to monitor and detect abuse - * see the openai docs for more details + * User is an optional unique identifier representing the end-user, which can help OpenAI to monitor and detect abuse + * see the openai docs for more details */ -public record OpenAiEmbeddingsTaskSettings(String modelId, @Nullable String user) implements TaskSettings { +public class OpenAiEmbeddingsTaskSettings implements TaskSettings { public static final String NAME = "openai_embeddings_task_settings"; - public static final String OLD_MODEL_ID_FIELD = "model"; - public static final String MODEL_ID = "model_id"; public static final String USER = "user"; - private static final String MODEL_DEPRECATION_MESSAGE = - "The openai [task_settings.model] field is deprecated. Please use [task_settings.model_id] instead."; - private static final Logger logger = LogManager.getLogger(OpenAiEmbeddingsTaskSettings.class); public static OpenAiEmbeddingsTaskSettings fromMap(Map map, OpenAiParseContext context) { ValidationException validationException = new ValidationException(); - String oldModelId = extractOptionalString(map, OLD_MODEL_ID_FIELD, ModelConfigurations.TASK_SETTINGS, validationException); - logOldModelDeprecation(oldModelId, context, logger); - - String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.TASK_SETTINGS, validationException); String user = extractOptionalString(map, USER, ModelConfigurations.TASK_SETTINGS, validationException); - - var modelIdToUse = getModelId(oldModelId, modelId, validationException); - if (validationException.validationErrors().isEmpty() == false) { throw validationException; } - return new OpenAiEmbeddingsTaskSettings(modelIdToUse, user); - } - - // default for testing - static void logOldModelDeprecation(@Nullable String oldModelId, OpenAiParseContext context, Logger logger) { - if (OpenAiParseContext.isRequestContext(context) && oldModelId != null) { - logger.info(MODEL_DEPRECATION_MESSAGE); - } - } - - private static String getModelId(@Nullable String oldModelId, @Nullable String modelId, ValidationException validationException) { - var modelIdToUse = modelId != null ? modelId : oldModelId; - - if (modelIdToUse == null) { - validationException.addValidationError(ServiceUtils.missingSettingErrorMsg(MODEL_ID, ModelConfigurations.TASK_SETTINGS)); - } - - return modelIdToUse; + return new OpenAiEmbeddingsTaskSettings(user); } /** @@ -90,24 +57,28 @@ public static OpenAiEmbeddingsTaskSettings of( OpenAiEmbeddingsTaskSettings originalSettings, OpenAiEmbeddingsRequestTaskSettings requestSettings ) { - var modelToUse = requestSettings.modelId() == null ? originalSettings.modelId : requestSettings.modelId(); var userToUse = requestSettings.user() == null ? originalSettings.user : requestSettings.user(); - - return new OpenAiEmbeddingsTaskSettings(modelToUse, userToUse); + return new OpenAiEmbeddingsTaskSettings(userToUse); } - public OpenAiEmbeddingsTaskSettings { - Objects.requireNonNull(modelId); + private final String user; + + public OpenAiEmbeddingsTaskSettings(@Nullable String user) { + this.user = user; } public OpenAiEmbeddingsTaskSettings(StreamInput in) throws IOException { - this(in.readString(), in.readOptionalString()); + if (in.getTransportVersion().onOrAfter(TransportVersions.ML_MODEL_IN_SERVICE_SETTINGS)) { + this.user = in.readOptionalString(); + } else { + var discard = in.readString(); + this.user = in.readOptionalString(); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(MODEL_ID, modelId); if (user != null) { builder.field(USER, user); } @@ -115,6 +86,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public String user() { + return user; + } + @Override public String getWriteableName() { return NAME; @@ -127,7 +102,24 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(modelId); - out.writeOptionalString(user); + if (out.getTransportVersion().onOrAfter(TransportVersions.ML_MODEL_IN_SERVICE_SETTINGS)) { + out.writeOptionalString(user); + } else { + out.writeString("m"); // write any string + out.writeOptionalString(user); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OpenAiEmbeddingsTaskSettings that = (OpenAiEmbeddingsTaskSettings) o; + return Objects.equals(user, that.user); + } + + @Override + public int hashCode() { + return Objects.hash(user); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java index 23b6f1ea2fbe3..cf1a569548143 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java @@ -92,7 +92,7 @@ public void testCreate_OpenAiEmbeddingsModel() throws IOException { var model = createModel(getUrl(webServer), "org", "secret", "model", "user"); var actionCreator = new OpenAiActionCreator(sender, createWithEmptySettings(threadPool)); - var overriddenTaskSettings = getRequestTaskSettingsMap(null, "overridden_user"); + var overriddenTaskSettings = getRequestTaskSettingsMap("overridden_user"); var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); @@ -161,7 +161,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From413StatusC var model = createModel(getUrl(webServer), "org", "secret", "model", "user"); var actionCreator = new OpenAiActionCreator(sender, createWithEmptySettings(threadPool)); - var overriddenTaskSettings = getRequestTaskSettingsMap(null, "overridden_user"); + var overriddenTaskSettings = getRequestTaskSettingsMap("overridden_user"); var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); @@ -244,7 +244,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From400StatusC var model = createModel(getUrl(webServer), "org", "secret", "model", "user"); var actionCreator = new OpenAiActionCreator(sender, createWithEmptySettings(threadPool)); - var overriddenTaskSettings = getRequestTaskSettingsMap(null, "overridden_user"); + var overriddenTaskSettings = getRequestTaskSettingsMap("overridden_user"); var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); @@ -312,7 +312,7 @@ public void testExecute_TruncatesInputBeforeSending() throws IOException { // truncated to 1 token = 3 characters var model = createModel(getUrl(webServer), "org", "secret", "model", "user", 1); var actionCreator = new OpenAiActionCreator(sender, createWithEmptySettings(threadPool)); - var overriddenTaskSettings = getRequestTaskSettingsMap(null, "overridden_user"); + var overriddenTaskSettings = getRequestTaskSettingsMap("overridden_user"); var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index cf2440afd77a7..3fd4d17d7a6e4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderFactory; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.ServiceFields; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModelTests; import org.hamcrest.MatcherAssert; @@ -60,6 +61,7 @@ import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -103,7 +105,7 @@ public void testParseRequestConfig_CreatesAnOpenAiEmbeddingsModel() throws IOExc var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); }, exception -> fail("Unexpected exception: " + exception)); @@ -112,8 +114,8 @@ public void testParseRequestConfig_CreatesAnOpenAiEmbeddingsModel() throws IOExc "id", TaskType.TEXT_EMBEDDING, getRequestConfigMap( - getServiceSettingsMap("url", "org"), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org"), + getTaskSettingsMap("user"), getSecretSettingsMap("secret") ), Set.of(), @@ -141,8 +143,8 @@ public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOExcepti "id", TaskType.SPARSE_EMBEDDING, getRequestConfigMap( - getServiceSettingsMap("url", "org"), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org"), + getTaskSettingsMap("user"), getSecretSettingsMap("secret") ), Set.of(), @@ -159,8 +161,8 @@ public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws I ) ) { var config = getRequestConfigMap( - getServiceSettingsMap("url", "org"), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org"), + getTaskSettingsMap("user"), getSecretSettingsMap("secret") ); config.put("extra_key", "value"); @@ -187,10 +189,10 @@ public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInServiceSettingsMa new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var serviceSettings = getServiceSettingsMap("url", "org"); + var serviceSettings = getServiceSettingsMap("model", "url", "org"); serviceSettings.put("extra_key", "value"); - var config = getRequestConfigMap(serviceSettings, getTaskSettingsMap("model", "user"), getSecretSettingsMap("secret")); + var config = getRequestConfigMap(serviceSettings, getTaskSettingsMap("user"), getSecretSettingsMap("secret")); ActionListener modelVerificationListener = ActionListener.wrap((model) -> { fail("Expected exception, but got model: " + model); @@ -210,10 +212,10 @@ public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInTaskSettingsMap() new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var taskSettingsMap = getTaskSettingsMap("model", "user"); + var taskSettingsMap = getTaskSettingsMap("user"); taskSettingsMap.put("extra_key", "value"); - var config = getRequestConfigMap(getServiceSettingsMap("url", "org"), taskSettingsMap, getSecretSettingsMap("secret")); + var config = getRequestConfigMap(getServiceSettingsMap("model", "url", "org"), taskSettingsMap, getSecretSettingsMap("secret")); ActionListener modelVerificationListener = ActionListener.wrap((model) -> { fail("Expected exception, but got model: " + model); @@ -236,7 +238,7 @@ public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInSecretSettingsMap var secretSettingsMap = getSecretSettingsMap("secret"); secretSettingsMap.put("extra_key", "value"); - var config = getRequestConfigMap(getServiceSettingsMap("url", "org"), getTaskSettingsMap("model", "user"), secretSettingsMap); + var config = getRequestConfigMap(getServiceSettingsMap("model", "url", "org"), getTaskSettingsMap("user"), secretSettingsMap); ActionListener modelVerificationListener = ActionListener.wrap((model) -> { fail("Expected exception, but got model: " + model); @@ -263,7 +265,7 @@ public void testParseRequestConfig_CreatesAnOpenAiEmbeddingsModelWithoutUserUrlO var embeddingsModel = (OpenAiEmbeddingsModel) model; assertNull(embeddingsModel.getServiceSettings().uri()); assertNull(embeddingsModel.getServiceSettings().organizationId()); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertNull(embeddingsModel.getTaskSettings().user()); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); }, exception -> fail("Unexpected exception: " + exception)); @@ -271,7 +273,39 @@ public void testParseRequestConfig_CreatesAnOpenAiEmbeddingsModelWithoutUserUrlO service.parseRequestConfig( "id", TaskType.TEXT_EMBEDDING, - getRequestConfigMap(getServiceSettingsMap(null, null), getTaskSettingsMap("model", null), getSecretSettingsMap("secret")), + getRequestConfigMap(getServiceSettingsMap("model", null, null), getTaskSettingsMap(null), getSecretSettingsMap("secret")), + Set.of(), + modelVerificationListener + ); + } + } + + public void testParseRequestConfig_MovesModel() throws IOException { + try ( + var service = new OpenAiService( + new SetOnce<>(mock(HttpRequestSenderFactory.class)), + new SetOnce<>(createWithEmptySettings(threadPool)) + ) + ) { + ActionListener modelVerificationListener = ActionListener.wrap(model -> { + assertThat(model, instanceOf(OpenAiEmbeddingsModel.class)); + + var embeddingsModel = (OpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); + assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, exception -> fail("Unexpected exception: " + exception)); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + getServiceSettingsMap("model", "url", "org"), + getTaskSettingsMap("user"), + getSecretSettingsMap("secret") + ), Set.of(), modelVerificationListener ); @@ -286,8 +320,8 @@ public void testParsePersistedConfigWithSecrets_CreatesAnOpenAiEmbeddingsModel() ) ) { var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org", 100, false), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org", 100, false), + getTaskSettingsMap("user"), getSecretSettingsMap("secret") ); @@ -303,7 +337,7 @@ public void testParsePersistedConfigWithSecrets_CreatesAnOpenAiEmbeddingsModel() var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); } @@ -317,8 +351,8 @@ public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidM ) ) { var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org"), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org"), + getTaskSettingsMap("user"), getSecretSettingsMap("secret") ); @@ -347,8 +381,8 @@ public void testParsePersistedConfigWithSecrets_CreatesAnOpenAiEmbeddingsModelWi ) ) { var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap(null, null, null, true), - getTaskSettingsMap("model", null), + getServiceSettingsMap("model", null, null, null, true), + getTaskSettingsMap(null), getSecretSettingsMap("secret") ); @@ -364,7 +398,7 @@ public void testParsePersistedConfigWithSecrets_CreatesAnOpenAiEmbeddingsModelWi var embeddingsModel = (OpenAiEmbeddingsModel) model; assertNull(embeddingsModel.getServiceSettings().uri()); assertNull(embeddingsModel.getServiceSettings().organizationId()); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertNull(embeddingsModel.getTaskSettings().user()); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); } @@ -378,8 +412,8 @@ public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExists ) ) { var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org", null, true), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org", null, true), + getTaskSettingsMap("user"), getSecretSettingsMap("secret") ); persistedConfig.config().put("extra_key", "value"); @@ -394,9 +428,10 @@ public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExists assertThat(model, instanceOf(OpenAiEmbeddingsModel.class)); var embeddingsModel = (OpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); } @@ -413,8 +448,8 @@ public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExists secretSettingsMap.put("extra_key", "value"); var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org", null, true), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org", null, true), + getTaskSettingsMap("user"), secretSettingsMap ); @@ -430,7 +465,7 @@ public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExists var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); } @@ -444,8 +479,8 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSe ) ) { var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org", null, true), - getTaskSettingsMap("model", "user"), + getServiceSettingsMap("model", "url", "org", null, true), + getTaskSettingsMap("user"), getSecretSettingsMap("secret") ); persistedConfig.secrets.put("extra_key", "value"); @@ -460,9 +495,10 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSe assertThat(model, instanceOf(OpenAiEmbeddingsModel.class)); var embeddingsModel = (OpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); } @@ -475,14 +511,10 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSe new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var serviceSettingsMap = getServiceSettingsMap("url", "org", null, true); + var serviceSettingsMap = getServiceSettingsMap("model", "url", "org", null, true); serviceSettingsMap.put("extra_key", "value"); - var persistedConfig = getPersistedConfigMap( - serviceSettingsMap, - getTaskSettingsMap("model", "user"), - getSecretSettingsMap("secret") - ); + var persistedConfig = getPersistedConfigMap(serviceSettingsMap, getTaskSettingsMap("user"), getSecretSettingsMap("secret")); var model = service.parsePersistedConfigWithSecrets( "id", @@ -496,7 +528,7 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSe var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); } @@ -509,11 +541,11 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInTa new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var taskSettingsMap = getTaskSettingsMap("model", "user"); + var taskSettingsMap = getTaskSettingsMap("user"); taskSettingsMap.put("extra_key", "value"); var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org", null, true), + getServiceSettingsMap("model", "url", "org", null, true), taskSettingsMap, getSecretSettingsMap("secret") ); @@ -530,7 +562,7 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInTa var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); } @@ -544,8 +576,8 @@ public void testParsePersistedConfig_CreatesAnOpenAiEmbeddingsModel() throws IOE ) ) { var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org", null, true), - getTaskSettingsMap("model", "user") + getServiceSettingsMap("model", "url", "org", null, true), + getTaskSettingsMap("user") ); var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); @@ -555,7 +587,7 @@ public void testParsePersistedConfig_CreatesAnOpenAiEmbeddingsModel() throws IOE var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertNull(embeddingsModel.getSecretSettings()); } @@ -568,7 +600,7 @@ public void testParsePersistedConfig_ThrowsErrorTryingToParseInvalidModel() thro new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var persistedConfig = getPersistedConfigMap(getServiceSettingsMap("url", "org"), getTaskSettingsMap("model", "user")); + var persistedConfig = getPersistedConfigMap(getServiceSettingsMap("model", "url", "org"), getTaskSettingsMap("user")); var thrownException = expectThrows( ElasticsearchStatusException.class, @@ -589,7 +621,7 @@ public void testParsePersistedConfig_CreatesAnOpenAiEmbeddingsModelWithoutUserUr new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var persistedConfig = getPersistedConfigMap(getServiceSettingsMap(null, null, null, true), getTaskSettingsMap("model", null)); + var persistedConfig = getPersistedConfigMap(getServiceSettingsMap("model", null, null, null, true), getTaskSettingsMap(null)); var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); @@ -598,7 +630,7 @@ public void testParsePersistedConfig_CreatesAnOpenAiEmbeddingsModelWithoutUserUr var embeddingsModel = (OpenAiEmbeddingsModel) model; assertNull(embeddingsModel.getServiceSettings().uri()); assertNull(embeddingsModel.getServiceSettings().organizationId()); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertNull(embeddingsModel.getTaskSettings().user()); assertNull(embeddingsModel.getSecretSettings()); } @@ -612,8 +644,8 @@ public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInConfig() ) ) { var persistedConfig = getPersistedConfigMap( - getServiceSettingsMap("url", "org", null, true), - getTaskSettingsMap("model", "user") + getServiceSettingsMap("model", "url", "org", null, true), + getTaskSettingsMap("user") ); persistedConfig.config().put("extra_key", "value"); @@ -624,7 +656,7 @@ public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInConfig() var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertNull(embeddingsModel.getSecretSettings()); } @@ -637,10 +669,10 @@ public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInServiceSettin new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var serviceSettingsMap = getServiceSettingsMap("url", "org", null, true); + var serviceSettingsMap = getServiceSettingsMap("model", "url", "org", null, true); serviceSettingsMap.put("extra_key", "value"); - var persistedConfig = getPersistedConfigMap(serviceSettingsMap, getTaskSettingsMap("model", "user")); + var persistedConfig = getPersistedConfigMap(serviceSettingsMap, getTaskSettingsMap("user")); var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); @@ -649,7 +681,7 @@ public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInServiceSettin var embeddingsModel = (OpenAiEmbeddingsModel) model; assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertNull(embeddingsModel.getSecretSettings()); } @@ -662,19 +694,20 @@ public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInTaskSettings( new SetOnce<>(createWithEmptySettings(threadPool)) ) ) { - var taskSettingsMap = getTaskSettingsMap("model", "user"); + var taskSettingsMap = getTaskSettingsMap("user"); taskSettingsMap.put("extra_key", "value"); - var persistedConfig = getPersistedConfigMap(getServiceSettingsMap("url", "org", null, true), taskSettingsMap); + var persistedConfig = getPersistedConfigMap(getServiceSettingsMap("model", "url", "org", null, true), taskSettingsMap); var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); assertThat(model, instanceOf(OpenAiEmbeddingsModel.class)); var embeddingsModel = (OpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getServiceSettings().uri().toString(), is("url")); assertThat(embeddingsModel.getServiceSettings().organizationId(), is("org")); - assertThat(embeddingsModel.getTaskSettings().modelId(), is("model")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("model")); assertThat(embeddingsModel.getTaskSettings().user(), is("user")); assertNull(embeddingsModel.getSecretSettings()); } @@ -1058,6 +1091,33 @@ public void testInfer_UnauthorisedResponse() throws IOException { } } + public void testMoveModelFromTaskToServiceSettings() { + var taskSettings = new HashMap(); + taskSettings.put(ServiceFields.MODEL_ID, "model"); + var serviceSettings = new HashMap(); + OpenAiService.moveModelFromTaskToServiceSettings(taskSettings, serviceSettings); + assertThat(taskSettings.keySet(), empty()); + assertEquals("model", serviceSettings.get(ServiceFields.MODEL_ID)); + } + + public void testMoveModelFromTaskToServiceSettings_OldID() { + var taskSettings = new HashMap(); + taskSettings.put("model", "model"); + var serviceSettings = new HashMap(); + OpenAiService.moveModelFromTaskToServiceSettings(taskSettings, serviceSettings); + assertThat(taskSettings.keySet(), empty()); + assertEquals("model", serviceSettings.get(ServiceFields.MODEL_ID)); + } + + public void testMoveModelFromTaskToServiceSettings_AlreadyMoved() { + var taskSettings = new HashMap(); + var serviceSettings = new HashMap(); + taskSettings.put(ServiceFields.MODEL_ID, "model"); + OpenAiService.moveModelFromTaskToServiceSettings(taskSettings, serviceSettings); + assertThat(taskSettings.keySet(), empty()); + assertEquals("model", serviceSettings.get(ServiceFields.MODEL_ID)); + } + private Map getRequestConfigMap( Map serviceSettings, Map taskSettings, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java index 1d82f94b52b20..60ed5a13d9c58 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java @@ -24,7 +24,7 @@ public class OpenAiEmbeddingsModelTests extends ESTestCase { public void testOverrideWith_OverridesUser() { var model = createModel("url", "org", "api_key", "model_name", null); - var requestTaskSettingsMap = getRequestTaskSettingsMap(null, "user_override"); + var requestTaskSettingsMap = getRequestTaskSettingsMap("user_override"); var overriddenModel = OpenAiEmbeddingsModel.of(model, requestTaskSettingsMap); @@ -58,8 +58,8 @@ public static OpenAiEmbeddingsModel createModel( "id", TaskType.TEXT_EMBEDDING, "service", - new OpenAiEmbeddingsServiceSettings(url, org, SimilarityMeasure.DOT_PRODUCT, 1536, null, false), - new OpenAiEmbeddingsTaskSettings(modelName, user), + new OpenAiEmbeddingsServiceSettings(modelName, url, org, SimilarityMeasure.DOT_PRODUCT, 1536, null, false), + new OpenAiEmbeddingsTaskSettings(user), new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) ); } @@ -76,8 +76,8 @@ public static OpenAiEmbeddingsModel createModel( "id", TaskType.TEXT_EMBEDDING, "service", - new OpenAiEmbeddingsServiceSettings(url, org, SimilarityMeasure.DOT_PRODUCT, 1536, tokenLimit, false), - new OpenAiEmbeddingsTaskSettings(modelName, user), + new OpenAiEmbeddingsServiceSettings(modelName, url, org, SimilarityMeasure.DOT_PRODUCT, 1536, tokenLimit, false), + new OpenAiEmbeddingsTaskSettings(user), new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) ); } @@ -95,8 +95,8 @@ public static OpenAiEmbeddingsModel createModel( "id", TaskType.TEXT_EMBEDDING, "service", - new OpenAiEmbeddingsServiceSettings(url, org, SimilarityMeasure.DOT_PRODUCT, dimensions, tokenLimit, false), - new OpenAiEmbeddingsTaskSettings(modelName, user), + new OpenAiEmbeddingsServiceSettings(modelName, url, org, SimilarityMeasure.DOT_PRODUCT, dimensions, tokenLimit, false), + new OpenAiEmbeddingsTaskSettings(user), new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) ); } @@ -116,8 +116,8 @@ public static OpenAiEmbeddingsModel createModel( "id", TaskType.TEXT_EMBEDDING, "service", - new OpenAiEmbeddingsServiceSettings(url, org, similarityMeasure, dimensions, tokenLimit, dimensionsSetByUser), - new OpenAiEmbeddingsTaskSettings(modelName, user), + new OpenAiEmbeddingsServiceSettings(modelName, url, org, similarityMeasure, dimensions, tokenLimit, dimensionsSetByUser), + new OpenAiEmbeddingsTaskSettings(user), new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettingsTests.java index 7f62e0b7efe3b..5a39fcb61ff0a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsRequestTaskSettingsTests.java @@ -18,70 +18,22 @@ public class OpenAiEmbeddingsRequestTaskSettingsTests extends ESTestCase { public void testFromMap_ReturnsEmptySettings_WhenTheMapIsEmpty() { var settings = OpenAiEmbeddingsRequestTaskSettings.fromMap(new HashMap<>(Map.of())); - - assertNull(settings.modelId()); assertNull(settings.user()); } public void testFromMap_ReturnsEmptySettings_WhenTheMapDoesNotContainTheFields() { var settings = OpenAiEmbeddingsRequestTaskSettings.fromMap(new HashMap<>(Map.of("key", "model"))); - - assertNull(settings.modelId()); assertNull(settings.user()); } - public void testFromMap_ReturnsEmptyModel_WhenTheMapDoesNotContainThatField() { + public void testFromMap_ReturnsUser() { var settings = OpenAiEmbeddingsRequestTaskSettings.fromMap(new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.USER, "user"))); - - assertNull(settings.modelId()); assertThat(settings.user(), is("user")); } - public void testFromMap_ReturnsEmptyUser_WhenTheDoesMapNotContainThatField() { - var settings = OpenAiEmbeddingsRequestTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model")) - ); - - assertNull(settings.user()); - assertThat(settings.modelId(), is("model")); - } - - public void testFromMap_PrefersModelId_OverModel() { - var settings = OpenAiEmbeddingsRequestTaskSettings.fromMap( - new HashMap<>( - Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model", OpenAiEmbeddingsTaskSettings.MODEL_ID, "model_id") - ) - ); - - assertNull(settings.user()); - assertThat(settings.modelId(), is("model_id")); - } - - public static Map getRequestTaskSettingsMap(@Nullable String model, @Nullable String user) { - var map = new HashMap(); - - if (model != null) { - map.put(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, model); - } - - if (user != null) { - map.put(OpenAiEmbeddingsTaskSettings.USER, user); - } - - return map; - } - - public static Map getRequestTaskSettingsMap(@Nullable String model, @Nullable String modelId, @Nullable String user) { + public static Map getRequestTaskSettingsMap(@Nullable String user) { var map = new HashMap(); - if (model != null) { - map.put(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, model); - } - - if (modelId != null) { - map.put(OpenAiEmbeddingsTaskSettings.MODEL_ID, model); - } - if (user != null) { map.put(OpenAiEmbeddingsTaskSettings.USER, user); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java index 7250d3c19447b..51069b46afb94 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java @@ -44,6 +44,7 @@ public static OpenAiEmbeddingsServiceSettings createRandom() { } private static OpenAiEmbeddingsServiceSettings createRandom(String url) { + var modelId = randomAlphaOfLength(8); var organizationId = randomBoolean() ? randomAlphaOfLength(15) : null; SimilarityMeasure similarityMeasure = null; Integer dims = null; @@ -54,6 +55,7 @@ private static OpenAiEmbeddingsServiceSettings createRandom(String url) { } Integer maxInputTokens = randomBoolean() ? null : randomIntBetween(128, 256); return new OpenAiEmbeddingsServiceSettings( + modelId, ServiceUtils.createUri(url), organizationId, similarityMeasure, @@ -64,6 +66,7 @@ private static OpenAiEmbeddingsServiceSettings createRandom(String url) { } public void testFromMap_Request_CreatesSettingsCorrectly() { + var modelId = "model-foo"; var url = "https://www.abc.com"; var org = "organization"; var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); @@ -72,6 +75,8 @@ public void testFromMap_Request_CreatesSettingsCorrectly() { var serviceSettings = OpenAiEmbeddingsServiceSettings.fromMap( new HashMap<>( Map.of( + ServiceFields.MODEL_ID, + modelId, ServiceFields.URL, url, OpenAiEmbeddingsServiceSettings.ORGANIZATION, @@ -91,6 +96,7 @@ public void testFromMap_Request_CreatesSettingsCorrectly() { serviceSettings, is( new OpenAiEmbeddingsServiceSettings( + modelId, ServiceUtils.createUri(url), org, SimilarityMeasure.DOT_PRODUCT, @@ -103,6 +109,7 @@ public void testFromMap_Request_CreatesSettingsCorrectly() { } public void testFromMap_Request_DimensionsSetByUser_IsFalse_WhenDimensionsAreNotPresent() { + var modelId = "model-foo"; var url = "https://www.abc.com"; var org = "organization"; var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); @@ -110,6 +117,8 @@ public void testFromMap_Request_DimensionsSetByUser_IsFalse_WhenDimensionsAreNot var serviceSettings = OpenAiEmbeddingsServiceSettings.fromMap( new HashMap<>( Map.of( + ServiceFields.MODEL_ID, + modelId, ServiceFields.URL, url, OpenAiEmbeddingsServiceSettings.ORGANIZATION, @@ -127,6 +136,7 @@ public void testFromMap_Request_DimensionsSetByUser_IsFalse_WhenDimensionsAreNot serviceSettings, is( new OpenAiEmbeddingsServiceSettings( + modelId, ServiceUtils.createUri(url), org, SimilarityMeasure.DOT_PRODUCT, @@ -139,6 +149,7 @@ public void testFromMap_Request_DimensionsSetByUser_IsFalse_WhenDimensionsAreNot } public void testFromMap_Persistent_CreatesSettingsCorrectly() { + var modelId = "model-foo"; var url = "https://www.abc.com"; var org = "organization"; var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); @@ -147,6 +158,8 @@ public void testFromMap_Persistent_CreatesSettingsCorrectly() { var serviceSettings = OpenAiEmbeddingsServiceSettings.fromMap( new HashMap<>( Map.of( + ServiceFields.MODEL_ID, + modelId, ServiceFields.URL, url, OpenAiEmbeddingsServiceSettings.ORGANIZATION, @@ -168,6 +181,7 @@ public void testFromMap_Persistent_CreatesSettingsCorrectly() { serviceSettings, is( new OpenAiEmbeddingsServiceSettings( + modelId, ServiceUtils.createUri(url), org, SimilarityMeasure.DOT_PRODUCT, @@ -181,17 +195,20 @@ public void testFromMap_Persistent_CreatesSettingsCorrectly() { public void testFromMap_PersistentContext_DoesNotThrowException_WhenDimensionsIsNull() { var settings = OpenAiEmbeddingsServiceSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsServiceSettings.DIMENSIONS_SET_BY_USER, true)), + new HashMap<>(Map.of(OpenAiEmbeddingsServiceSettings.DIMENSIONS_SET_BY_USER, true, ServiceFields.MODEL_ID, "m")), OpenAiParseContext.PERSISTENT ); - assertThat(settings, is(new OpenAiEmbeddingsServiceSettings((URI) null, null, null, null, null, true))); + assertThat(settings, is(new OpenAiEmbeddingsServiceSettings("m", (URI) null, null, null, null, null, true))); } public void testFromMap_PersistentContext_ThrowsException_WhenDimensionsSetByUserIsNull() { var exception = expectThrows( ValidationException.class, - () -> OpenAiEmbeddingsServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.DIMENSIONS, 1)), OpenAiParseContext.PERSISTENT) + () -> OpenAiEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.DIMENSIONS, 1, ServiceFields.MODEL_ID, "m")), + OpenAiParseContext.PERSISTENT + ) ); assertThat( @@ -202,17 +219,21 @@ public void testFromMap_PersistentContext_ThrowsException_WhenDimensionsSetByUse public void testFromMap_MissingUrl_DoesNotThrowException() { var serviceSettings = OpenAiEmbeddingsServiceSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsServiceSettings.ORGANIZATION, "org")), + new HashMap<>(Map.of(ServiceFields.MODEL_ID, "m", OpenAiEmbeddingsServiceSettings.ORGANIZATION, "org")), OpenAiParseContext.REQUEST ); assertNull(serviceSettings.uri()); + assertThat(serviceSettings.modelId(), is("m")); assertThat(serviceSettings.organizationId(), is("org")); } public void testFromMap_EmptyUrl_ThrowsError() { var thrownException = expectThrows( ValidationException.class, - () -> OpenAiEmbeddingsServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, "")), OpenAiParseContext.REQUEST) + () -> OpenAiEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.URL, "", ServiceFields.MODEL_ID, "m")), + OpenAiParseContext.REQUEST + ) ); assertThat( @@ -227,7 +248,10 @@ public void testFromMap_EmptyUrl_ThrowsError() { } public void testFromMap_MissingOrganization_DoesNotThrowException() { - var serviceSettings = OpenAiEmbeddingsServiceSettings.fromMap(new HashMap<>(), OpenAiParseContext.REQUEST); + var serviceSettings = OpenAiEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, "m")), + OpenAiParseContext.REQUEST + ); assertNull(serviceSettings.uri()); assertNull(serviceSettings.organizationId()); } @@ -236,7 +260,7 @@ public void testFromMap_EmptyOrganization_ThrowsError() { var thrownException = expectThrows( ValidationException.class, () -> OpenAiEmbeddingsServiceSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsServiceSettings.ORGANIZATION, "")), + new HashMap<>(Map.of(OpenAiEmbeddingsServiceSettings.ORGANIZATION, "", ServiceFields.MODEL_ID, "m")), OpenAiParseContext.REQUEST ) ); @@ -256,7 +280,10 @@ public void testFromMap_InvalidUrl_ThrowsError() { var url = "https://www.abc^.com"; var thrownException = expectThrows( ValidationException.class, - () -> OpenAiEmbeddingsServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, url)), OpenAiParseContext.REQUEST) + () -> OpenAiEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.URL, url, ServiceFields.MODEL_ID, "m")), + OpenAiParseContext.REQUEST + ) ); assertThat( @@ -270,7 +297,7 @@ public void testFromMap_InvalidSimilarity_ThrowsError() { var thrownException = expectThrows( ValidationException.class, () -> OpenAiEmbeddingsServiceSettings.fromMap( - new HashMap<>(Map.of(ServiceFields.SIMILARITY, similarity)), + new HashMap<>(Map.of(ServiceFields.SIMILARITY, similarity, ServiceFields.MODEL_ID, "m")), OpenAiParseContext.REQUEST ) ); @@ -279,41 +306,41 @@ public void testFromMap_InvalidSimilarity_ThrowsError() { } public void testToXContent_WritesDimensionsSetByUserTrue() throws IOException { - var entity = new OpenAiEmbeddingsServiceSettings("url", "org", null, null, null, true); + var entity = new OpenAiEmbeddingsServiceSettings("model", "url", "org", null, null, null, true); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); String xContentResult = Strings.toString(builder); assertThat(xContentResult, CoreMatchers.is(""" - {"url":"url","organization_id":"org","dimensions_set_by_user":true}""")); + {"model_id":"model","url":"url","organization_id":"org","dimensions_set_by_user":true}""")); } public void testToXContent_WritesDimensionsSetByUserFalse() throws IOException { - var entity = new OpenAiEmbeddingsServiceSettings("url", "org", null, null, null, false); + var entity = new OpenAiEmbeddingsServiceSettings("model", "url", "org", null, null, null, false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); String xContentResult = Strings.toString(builder); assertThat(xContentResult, CoreMatchers.is(""" - {"url":"url","organization_id":"org","dimensions_set_by_user":false}""")); + {"model_id":"model","url":"url","organization_id":"org","dimensions_set_by_user":false}""")); } public void testToXContent_WritesAllValues() throws IOException { - var entity = new OpenAiEmbeddingsServiceSettings("url", "org", SimilarityMeasure.DOT_PRODUCT, 1, 2, false); + var entity = new OpenAiEmbeddingsServiceSettings("model", "url", "org", SimilarityMeasure.DOT_PRODUCT, 1, 2, false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); String xContentResult = Strings.toString(builder); assertThat(xContentResult, CoreMatchers.is(""" - {"url":"url","organization_id":"org","similarity":"dot_product",""" + """ + {"model_id":"model","url":"url","organization_id":"org","similarity":"dot_product",""" + """ "dimensions":1,"max_input_tokens":2,"dimensions_set_by_user":false}""")); } public void testToFilteredXContent_WritesAllValues_ExceptDimensionsSetByUser() throws IOException { - var entity = new OpenAiEmbeddingsServiceSettings("url", "org", SimilarityMeasure.DOT_PRODUCT, 1, 2, false); + var entity = new OpenAiEmbeddingsServiceSettings("model", "url", "org", SimilarityMeasure.DOT_PRODUCT, 1, 2, false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); var filteredXContent = entity.getFilteredXContentObject(); @@ -321,7 +348,7 @@ public void testToFilteredXContent_WritesAllValues_ExceptDimensionsSetByUser() t String xContentResult = Strings.toString(builder); assertThat(xContentResult, CoreMatchers.is(""" - {"url":"url","organization_id":"org","similarity":"dot_product",""" + """ + {"model_id":"model","url":"url","organization_id":"org","similarity":"dot_product",""" + """ "dimensions":1,"max_input_tokens":2}""")); } @@ -340,9 +367,9 @@ protected OpenAiEmbeddingsServiceSettings mutateInstance(OpenAiEmbeddingsService return createRandomWithNonNullUrl(); } - public static Map getServiceSettingsMap(@Nullable String url, @Nullable String org) { + public static Map getServiceSettingsMap(String modelId, @Nullable String url, @Nullable String org) { var map = new HashMap(); - + map.put(ServiceFields.MODEL_ID, modelId); if (url != null) { map.put(ServiceFields.URL, url); } @@ -354,12 +381,14 @@ public static Map getServiceSettingsMap(@Nullable String url, @N } public static Map getServiceSettingsMap( + String model, @Nullable String url, @Nullable String org, @Nullable Integer dimensions, @Nullable Boolean dimensionsSetByUser ) { var map = new HashMap(); + map.put(ServiceFields.MODEL_ID, model); if (url != null) { map.put(ServiceFields.URL, url); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java index a5291cebf6ea3..3c91b68a545fd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java @@ -7,17 +7,12 @@ package org.elasticsearch.xpack.inference.services.openai.embeddings; -import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.services.openai.OpenAiParseContext; -import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import java.io.IOException; @@ -25,17 +20,11 @@ import java.util.Map; import static org.hamcrest.Matchers.is; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyNoMoreInteractions; public class OpenAiEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase { public static OpenAiEmbeddingsTaskSettings createRandomWithUser() { - return new OpenAiEmbeddingsTaskSettings(randomAlphaOfLength(15), randomAlphaOfLength(15)); + return new OpenAiEmbeddingsTaskSettings(randomAlphaOfLength(15)); } /** @@ -43,81 +32,42 @@ public static OpenAiEmbeddingsTaskSettings createRandomWithUser() { */ public static OpenAiEmbeddingsTaskSettings createRandom() { var user = randomBoolean() ? randomAlphaOfLength(15) : null; - return new OpenAiEmbeddingsTaskSettings(randomAlphaOfLength(15), user); + return new OpenAiEmbeddingsTaskSettings(user); } - public void testFromMap_MissingModel_ThrowException() { + public void testFromMap_WithUser() { + assertEquals( + new OpenAiEmbeddingsTaskSettings("user"), + OpenAiEmbeddingsTaskSettings.fromMap( + new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.USER, "user")), + OpenAiParseContext.REQUEST + ) + ); + } + + public void testFromMap_UserIsEmptyString() { var thrownException = expectThrows( ValidationException.class, () -> OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.USER, "user")), + new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.USER, "")), OpenAiParseContext.REQUEST ) ); MatcherAssert.assertThat( thrownException.getMessage(), - is( - Strings.format( - "Validation Failed: 1: [task_settings] does not contain the required setting [%s];", - OpenAiEmbeddingsTaskSettings.MODEL_ID - ) - ) - ); - } - - public void testFromMap_CreatesWithModelAndUser() { - var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model", OpenAiEmbeddingsTaskSettings.USER, "user")), - OpenAiParseContext.PERSISTENT - ); - - MatcherAssert.assertThat(taskSettings.modelId(), is("model")); - MatcherAssert.assertThat(taskSettings.user(), is("user")); - } - - public void testFromMap_CreatesWithModelId() { - var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.MODEL_ID, "model", OpenAiEmbeddingsTaskSettings.USER, "user")), - OpenAiParseContext.PERSISTENT - ); - - MatcherAssert.assertThat(taskSettings.modelId(), is("model")); - MatcherAssert.assertThat(taskSettings.user(), is("user")); - } - - public void testFromMap_PrefersModelId_OverModel() { - var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>( - Map.of( - OpenAiEmbeddingsTaskSettings.MODEL_ID, - "model", - OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, - "old_model", - OpenAiEmbeddingsTaskSettings.USER, - "user" - ) - ), - OpenAiParseContext.PERSISTENT + is(Strings.format("Validation Failed: 1: [task_settings] Invalid value empty string. [user] must be a non-empty string;")) ); - - MatcherAssert.assertThat(taskSettings.modelId(), is("model")); - MatcherAssert.assertThat(taskSettings.user(), is("user")); } public void testFromMap_MissingUser_DoesNotThrowException() { - var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model")), - OpenAiParseContext.PERSISTENT - ); - - MatcherAssert.assertThat(taskSettings.modelId(), is("model")); + var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of()), OpenAiParseContext.PERSISTENT); assertNull(taskSettings.user()); } public void testOverrideWith_KeepsOriginalValuesWithOverridesAreNull() { var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model", OpenAiEmbeddingsTaskSettings.USER, "user")), + new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.USER, "user")), OpenAiParseContext.PERSISTENT ); @@ -127,86 +77,16 @@ public void testOverrideWith_KeepsOriginalValuesWithOverridesAreNull() { public void testOverrideWith_UsesOverriddenSettings() { var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model", OpenAiEmbeddingsTaskSettings.USER, "user")), - OpenAiParseContext.PERSISTENT - ); - - var requestTaskSettings = OpenAiEmbeddingsRequestTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model2", OpenAiEmbeddingsTaskSettings.USER, "user2")) - ); - - var overriddenTaskSettings = OpenAiEmbeddingsTaskSettings.of(taskSettings, requestTaskSettings); - MatcherAssert.assertThat(overriddenTaskSettings, is(new OpenAiEmbeddingsTaskSettings("model2", "user2"))); - } - - public void testOverrideWith_UsesOverriddenSettings_UsesModel2_FromModelIdField() { - var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model", OpenAiEmbeddingsTaskSettings.USER, "user")), - OpenAiParseContext.PERSISTENT - ); - - var requestTaskSettings = OpenAiEmbeddingsRequestTaskSettings.fromMap( - new HashMap<>( - Map.of( - OpenAiEmbeddingsTaskSettings.MODEL_ID, - "model2", - OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, - "model3", - OpenAiEmbeddingsTaskSettings.USER, - "user2" - ) - ) - ); - - var overriddenTaskSettings = OpenAiEmbeddingsTaskSettings.of(taskSettings, requestTaskSettings); - MatcherAssert.assertThat(overriddenTaskSettings, is(new OpenAiEmbeddingsTaskSettings("model2", "user2"))); - } - - public void testOverrideWith_UsesOnlyNonNullModelSetting() { - var taskSettings = OpenAiEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model", OpenAiEmbeddingsTaskSettings.USER, "user")), + new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.USER, "user")), OpenAiParseContext.PERSISTENT ); var requestTaskSettings = OpenAiEmbeddingsRequestTaskSettings.fromMap( - new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, "model2")) + new HashMap<>(Map.of(OpenAiEmbeddingsTaskSettings.USER, "user2")) ); var overriddenTaskSettings = OpenAiEmbeddingsTaskSettings.of(taskSettings, requestTaskSettings); - MatcherAssert.assertThat(overriddenTaskSettings, is(new OpenAiEmbeddingsTaskSettings("model2", "user"))); - } - - public void testXContent_WritesModelId() throws IOException { - var entity = new OpenAiEmbeddingsTaskSettings("modelId", null); - - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - entity.toXContent(builder, null); - String xContentResult = Strings.toString(builder); - - assertThat(xContentResult, CoreMatchers.is(""" - {"model_id":"modelId"}""")); - } - - public void testLogModelDeprecation_CallsInfo_WhenContextIsRequest_AndOldModelIdIsDefined() { - var mockLogger = mock(Logger.class); - - OpenAiEmbeddingsTaskSettings.logOldModelDeprecation("model", OpenAiParseContext.REQUEST, mockLogger); - verify(mockLogger, times(1)).info(anyString()); - verifyNoMoreInteractions(mockLogger); - } - - public void testLogModelDeprecation_DoesNotCallInfo_WhenContextIsRequest_AndOldModelIdIsNull() { - var mockLogger = mock(Logger.class); - - OpenAiEmbeddingsTaskSettings.logOldModelDeprecation(null, OpenAiParseContext.PERSISTENT, mockLogger); - verifyNoInteractions(mockLogger); - } - - public void testLogModelDeprecation_DoesNotCallInfo_WhenContextIsPersistent_AndOldModelIdIsDefined() { - var mockLogger = mock(Logger.class); - - OpenAiEmbeddingsTaskSettings.logOldModelDeprecation("model", OpenAiParseContext.PERSISTENT, mockLogger); - verifyNoInteractions(mockLogger); + MatcherAssert.assertThat(overriddenTaskSettings, is(new OpenAiEmbeddingsTaskSettings("user2"))); } @Override @@ -224,8 +104,8 @@ protected OpenAiEmbeddingsTaskSettings mutateInstance(OpenAiEmbeddingsTaskSettin return createRandomWithUser(); } - public static Map getTaskSettingsMap(String model, @Nullable String user) { - var map = new HashMap(Map.of(OpenAiEmbeddingsTaskSettings.OLD_MODEL_ID_FIELD, model)); + public static Map getTaskSettingsMap(@Nullable String user) { + var map = new HashMap(); if (user != null) { map.put(OpenAiEmbeddingsTaskSettings.USER, user); From b984b44767f9316e956e1a3e128c168535a47149 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 13 Feb 2024 20:13:29 +0100 Subject: [PATCH 13/78] Update Gradle wrapper to 8.6 (#103796) --- .../gradle/wrapper/gradle-wrapper.properties | 4 ++-- .../src/main/resources/minimumGradleVersion | 2 +- gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew.bat | 20 +++++++++---------- .../gradle/wrapper/gradle-wrapper.properties | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index a7a990ab2a89e..865f1ba80d1e6 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=c16d517b50dd28b3f5838f0e844b7520b8f1eb610f2f29de7e4e04a1b7c9c79b -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip +distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index 3d512719cff9b..f043ef362390f 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.5 \ No newline at end of file +8.6 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a7a990ab2a89e..865f1ba80d1e6 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=c16d517b50dd28b3f5838f0e844b7520b8f1eb610f2f29de7e4e04a1b7c9c79b -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip +distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew.bat b/gradlew.bat index 6689b85beecde..7101f8e4676fc 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -43,11 +43,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -57,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index a7a990ab2a89e..865f1ba80d1e6 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=c16d517b50dd28b3f5838f0e844b7520b8f1eb610f2f29de7e4e04a1b7c9c79b -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip +distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 8e53b0f0bffd18e8ac09a2241b900dae98e954a7 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Tue, 13 Feb 2024 14:21:24 -0500 Subject: [PATCH 14/78] Cross cluster search minimize roundtrips avoids incremental merges until Kibana polls via async-search-status endpoint (#105455) This is a temporary change to avoid doing incremental merges in cross-cluster async-search when minimize_roundtrips=true. Currently, Kibana polls for status of the async-search via the _async_search endpoint, which (without this change) will do an incremental merge of all search results. Once Kibana moves to polling status via _async_search/status, then we will undo the change in this commit. --- docs/changelog/103134.yaml | 5 ----- .../xpack/search/MutableSearchResponse.java | 10 ++++++---- 2 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 docs/changelog/103134.yaml diff --git a/docs/changelog/103134.yaml b/docs/changelog/103134.yaml deleted file mode 100644 index 13bb0323645f5..0000000000000 --- a/docs/changelog/103134.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 103134 -summary: CCS with `minimize_roundtrips` performs incremental merges of each `SearchResponse` -area: Search -type: enhancement -issues: [] diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index 45f393a9a845e..a0aabae6e7218 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -306,10 +306,12 @@ synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, lon * (for local-only/CCS minimize_roundtrips=false) */ private SearchResponseMerger createSearchResponseMerger(AsyncSearchTask task) { - if (task.getSearchResponseMergerSupplier() == null) { - return null; // local search and CCS minimize_roundtrips=false - } - return task.getSearchResponseMergerSupplier().get(); + return null; + // TODO uncomment this code once Kibana moves to polling the _async_search/status endpoint to determine if a search is done + // if (task.getSearchResponseMergerSupplier() == null) { + // return null; // local search and CCS minimize_roundtrips=false + // } + // return task.getSearchResponseMergerSupplier().get(); } private SearchResponse getMergedResponse(SearchResponseMerger merger) { From c884945a9327b62a97146437a053274260230c9d Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 13 Feb 2024 15:11:47 -0600 Subject: [PATCH 15/78] Adding executedPipelines to the IngestDocument copy constructor (#105427) --- docs/changelog/105427.yaml | 5 ++++ .../rest-api-spec/test/ingest/90_simulate.yml | 11 +-------- .../elasticsearch/ingest/IngestDocument.java | 9 +++++++- .../ingest/IngestDocumentTests.java | 23 +++++++++++++++++++ 4 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/105427.yaml diff --git a/docs/changelog/105427.yaml b/docs/changelog/105427.yaml new file mode 100644 index 0000000000000..e73853b9dce92 --- /dev/null +++ b/docs/changelog/105427.yaml @@ -0,0 +1,5 @@ +pr: 105427 +summary: Adding `executedPipelines` to the `IngestDocument` copy constructor +area: Ingest Node +type: bug +issues: [] diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml index b54c21517ca22..8f69b6d565ad4 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml @@ -714,18 +714,9 @@ teardown: - do: ingest.simulate: verbose: true + id: "outer" body: > { - "pipeline": { - "processors" : [ - { - "pipeline" : { - "name": "outer" - } - } - ] - } - , "docs": [ { "_index": "index", diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index a915f197d0cdf..e8eb903194b5a 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -91,7 +91,7 @@ public IngestDocument(String index, String id, long version, String routing, Ver } // note: these rest of these constructors deal with the data-centric view of the IngestDocument, not the execution-centric view. - // For example, the copy constructor doesn't populate the `executedPipelines` or `indexHistory` (as well as some other fields), + // For example, the copy constructor doesn't populate the `indexHistory` (as well as some other fields), // because those fields are execution-centric. /** @@ -104,6 +104,13 @@ public IngestDocument(IngestDocument other) { new IngestCtxMap(deepCopyMap(ensureNoSelfReferences(other.ctxMap.getSource())), other.ctxMap.getMetadata().clone()), deepCopyMap(other.ingestMetadata) ); + /* + * The executedPipelines field is clearly execution-centric rather than data centric. Despite what the comment above says, we're + * copying it here anyway. THe reason is that this constructor is only called from two non-test locations, and both of those + * involve the simulate pipeline logic. The simulate pipeline logic needs this information. Rather than making the code more + * complicated, we're just copying this over here since it does no harm. + */ + this.executedPipelines.addAll(other.executedPipelines); } /** diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index 71e90e8f4cc06..2e3649ddbf859 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -1017,6 +1017,29 @@ public void testCopyConstructor() { } } + public void testCopyConstructorWithExecutedPipelines() { + /* + * This is similar to the first part of testCopyConstructor, except that we're executing a pipeilne, and running the + * assertions inside the processor so that we can test that executedPipelines is correct. + */ + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + TestProcessor processor = new TestProcessor(ingestDocument1 -> { + assertThat(ingestDocument1.getPipelineStack().size(), equalTo(1)); + IngestDocument copy = new IngestDocument(ingestDocument1); + assertThat(ingestDocument1.getSourceAndMetadata(), not(sameInstance(copy.getSourceAndMetadata()))); + assertThat(ingestDocument1.getCtxMap(), not(sameInstance(copy.getCtxMap()))); + assertThat(ingestDocument1.getCtxMap().getMetadata(), not(sameInstance(copy.getCtxMap().getMetadata()))); + assertIngestDocument(ingestDocument1, copy); + assertThat(copy.getPipelineStack(), equalTo(ingestDocument1.getPipelineStack())); + }); + Pipeline pipeline = new Pipeline("pipeline1", "test pipeline", 1, Map.of(), new CompoundProcessor(processor)); + ingestDocument.executePipeline(pipeline, (ingestDocument1, exception) -> { + assertNotNull(ingestDocument1); + assertNull(exception); + }); + assertThat(processor.getInvokedCounter(), equalTo(1)); + } + public void testCopyConstructorWithZonedDateTime() { ZoneId timezone = ZoneId.of("Europe/London"); From f0ec29438209792ba5e7d1f24aa5a8bd6d908a6a Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 13 Feb 2024 16:28:31 -0600 Subject: [PATCH 16/78] Limiting the number of nested pipelines that can be executed (#105428) Limiting the number of nested pipelines that can be executed within a single pipeline to 100 --- docs/changelog/105428.yaml | 5 ++ .../ingest/common/ManyNestedPipelinesIT.java | 80 +++++++++++++++---- .../elasticsearch/ingest/IngestDocument.java | 9 ++- .../ingest/TrackingResultProcessorTests.java | 54 ++++++++++++- 4 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 docs/changelog/105428.yaml diff --git a/docs/changelog/105428.yaml b/docs/changelog/105428.yaml new file mode 100644 index 0000000000000..49a80150b4303 --- /dev/null +++ b/docs/changelog/105428.yaml @@ -0,0 +1,5 @@ +pr: 105428 +summary: Limiting the number of nested pipelines that can be executed +area: Ingest Node +type: enhancement +issues: [] diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java index b80d34e11ebc2..c9f3f023b43ef 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Strings; +import org.elasticsearch.ingest.GraphStructureException; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -40,6 +41,9 @@ */ public class ManyNestedPipelinesIT extends ESIntegTestCase { private final int manyPipelinesCount = randomIntBetween(2, 50); + private final int tooManyPipelinesCount = IngestDocument.MAX_PIPELINES + 1; + private static final String MANY_PIPELINES_PREFIX = "many_"; + private static final String TOO_MANY_PIPELINES_PREFIX = "too_many_"; @Override protected Collection> nodePlugins() { @@ -50,19 +54,21 @@ protected Collection> nodePlugins() { public void loadManyPipelines() { internalCluster().ensureAtLeastNumDataNodes(1); internalCluster().startMasterOnlyNode(); - createChainedPipelines(manyPipelinesCount); + createManyChainedPipelines(); } public void testIngestManyPipelines() { String index = "index"; - DocWriteResponse response = prepareIndex(index).setSource(Map.of("foo", "bar")).setPipeline("pipeline_0").get(); + DocWriteResponse response = prepareIndex(index).setSource(Map.of("foo", "bar")) + .setPipeline(MANY_PIPELINES_PREFIX + "pipeline_0") + .get(); assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); GetResponse getREsponse = client().prepareGet(index, response.getId()).get(); assertThat(getREsponse.getSource().get("foo"), equalTo("baz")); } public void testSimulateManyPipelines() throws IOException { - List results = executeSimulate(false); + List results = executeSimulateManyPipelines(false); assertThat(results.size(), equalTo(1)); assertThat(results.get(0), instanceOf(SimulateDocumentBaseResult.class)); SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) results.get(0); @@ -72,7 +78,7 @@ public void testSimulateManyPipelines() throws IOException { } public void testSimulateVerboseManyPipelines() throws IOException { - List results = executeSimulate(true); + List results = executeSimulateManyPipelines(true); assertThat(results.size(), equalTo(1)); assertThat(results.get(0), instanceOf(SimulateDocumentVerboseResult.class)); SimulateDocumentVerboseResult result = (SimulateDocumentVerboseResult) results.get(0); @@ -83,7 +89,45 @@ public void testSimulateVerboseManyPipelines() throws IOException { assertThat(resultDoc.getFieldValue("foo", String.class), equalTo("baz")); } - private List executeSimulate(boolean verbose) throws IOException { + public void testTooManyPipelines() throws IOException { + /* + * Logically, this test method contains three tests (too many pipelines for ingest, simulate, and simulate verbose). But creating + * pipelines is so slow that they are lumped into this one method. + */ + createTooManyChainedPipelines(); + expectThrows( + GraphStructureException.class, + () -> prepareIndex("foo").setSource(Map.of("foo", "bar")).setPipeline(TOO_MANY_PIPELINES_PREFIX + "pipeline_0").get() + ); + { + List results = executeSimulateTooManyPipelines(false); + assertThat(results.size(), equalTo(1)); + assertThat(results.get(0), instanceOf(SimulateDocumentBaseResult.class)); + SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) results.get(0); + assertNotNull(result.getFailure()); + assertNotNull(result.getFailure().getCause()); + assertThat(result.getFailure().getCause(), instanceOf(GraphStructureException.class)); + } + { + List results = executeSimulateTooManyPipelines(true); + assertThat(results.size(), equalTo(1)); + assertThat(results.get(0), instanceOf(SimulateDocumentVerboseResult.class)); + SimulateDocumentVerboseResult result = (SimulateDocumentVerboseResult) results.get(0); + assertNotNull(result); + assertNotNull(result.getProcessorResults().get(0).getFailure()); + assertThat(result.getProcessorResults().get(0).getFailure().getCause(), instanceOf(GraphStructureException.class)); + } + } + + private List executeSimulateManyPipelines(boolean verbose) throws IOException { + return executeSimulatePipelines(MANY_PIPELINES_PREFIX, verbose); + } + + private List executeSimulateTooManyPipelines(boolean verbose) throws IOException { + return executeSimulatePipelines(TOO_MANY_PIPELINES_PREFIX, verbose); + } + + private List executeSimulatePipelines(String prefix, boolean verbose) throws IOException { BytesReference simulateRequestBytes = BytesReference.bytes( jsonBuilder().startObject() .startArray("docs") @@ -98,22 +142,30 @@ private List executeSimulate(boolean verbose) throws IOE .endObject() ); SimulatePipelineResponse simulatePipelineResponse = clusterAdmin().prepareSimulatePipeline(simulateRequestBytes, XContentType.JSON) - .setId("pipeline_0") + .setId(prefix + "pipeline_0") .setVerbose(verbose) .get(); return simulatePipelineResponse.getResults(); } - private void createChainedPipelines(int count) { + private void createManyChainedPipelines() { + createChainedPipelines(MANY_PIPELINES_PREFIX, manyPipelinesCount); + } + + private void createTooManyChainedPipelines() { + createChainedPipelines(TOO_MANY_PIPELINES_PREFIX, tooManyPipelinesCount); + } + + private void createChainedPipelines(String prefix, int count) { for (int i = 0; i < count - 1; i++) { - createChainedPipeline(i); + createChainedPipeline(prefix, i); } - createLastPipeline(count - 1); + createLastPipeline(prefix, count - 1); } - private void createChainedPipeline(int number) { - String pipelineId = "pipeline_" + number; - String nextPipelineId = "pipeline_" + (number + 1); + private void createChainedPipeline(String prefix, int number) { + String pipelineId = prefix + "pipeline_" + number; + String nextPipelineId = prefix + "pipeline_" + (number + 1); String pipelineTemplate = """ { "processors": [ @@ -129,8 +181,8 @@ private void createChainedPipeline(int number) { clusterAdmin().preparePutPipeline(pipelineId, new BytesArray(pipeline), XContentType.JSON).get(); } - private void createLastPipeline(int number) { - String pipelineId = "pipeline_" + number; + private void createLastPipeline(String prefix, int number) { + String pipelineId = prefix + "pipeline_" + number; String pipeline = """ { "processors": [ diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index e8eb903194b5a..cbfd5adc1abd2 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -51,6 +51,8 @@ public final class IngestDocument { private static final String PIPELINE_CYCLE_ERROR_MESSAGE = "Cycle detected for pipeline: "; static final String TIMESTAMP = "timestamp"; + // This is the maximum number of nested pipelines that can be within a pipeline. If there are more, we bail out with an error + public static final int MAX_PIPELINES = Integer.parseInt(System.getProperty("es.ingest.max_pipelines", "100")); private final IngestCtxMap ctxMap; private final Map ingestMetadata; @@ -836,7 +838,12 @@ public void executePipeline(Pipeline pipeline, BiConsumer= MAX_PIPELINES) { + handler.accept( + null, + new GraphStructureException("Too many nested pipelines. Cannot have more than " + MAX_PIPELINES + " nested pipelines") + ); + } else if (executedPipelines.add(pipeline.getId())) { Object previousPipeline = ingestMetadata.put("pipeline", pipeline.getId()); pipeline.execute(this, (result, e) -> { executedPipelines.remove(pipeline.getId()); diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index e209bd067701a..d358e524c6ff9 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -682,7 +682,7 @@ public void testActualPipelineProcessorNested() throws Exception { */ IngestService ingestService = createIngestService(); PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); - int pipelineCount = randomIntBetween(2, 150); + int pipelineCount = randomIntBetween(2, IngestDocument.MAX_PIPELINES); for (int i = 0; i < pipelineCount - 1; i++) { String pipelineId = "pipeline" + i; String nextPipelineId = "pipeline" + (i + 1); @@ -728,6 +728,58 @@ public void testActualPipelineProcessorNested() throws Exception { assertThat(resultList.get(resultList.size() - 1).getType(), equalTo(countCallsProcessor.getType())); } + public void testActualPipelineProcessorNestedTooManyPipelines() throws Exception { + /* + * This test creates a pipeline made up of many nested pipeline processors, ending in a processor that counts both how many times + * it is called for a given document (by updating a field on that document) and how many times it is called overall. + */ + IngestService ingestService = createIngestService(); + PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); + int pipelineCount = randomIntBetween(IngestDocument.MAX_PIPELINES + 1, 500); + for (int i = 0; i < pipelineCount - 1; i++) { + String pipelineId = "pipeline" + i; + String nextPipelineId = "pipeline" + (i + 1); + Map nextPipelineConfig = new HashMap<>(); + nextPipelineConfig.put("name", nextPipelineId); + Pipeline pipeline = new Pipeline( + pipelineId, + null, + null, + null, + new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig)) + ); + when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline); + } + + // The last pipeline calls the CountCallsProcessor rather than yet another pipeline processor: + String lastPipelineId = "pipeline" + (pipelineCount - 1); + CountCallsProcessor countCallsProcessor = new CountCallsProcessor(); + Pipeline lastPipeline = new Pipeline(lastPipelineId, null, null, null, new CompoundProcessor(countCallsProcessor)); + when(ingestService.getPipeline(lastPipelineId)).thenReturn(lastPipeline); + + String firstPipelineId = "pipeline0"; + Map firstPipelineConfig = new HashMap<>(); + firstPipelineConfig.put("name", firstPipelineId); + PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig); + CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor); + + CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList); + + IngestDocument[] documentHolder = new IngestDocument[1]; + Exception[] exceptionHolder = new Exception[1]; + trackingProcessor.execute(ingestDocument, (result, e) -> { + documentHolder[0] = result; + exceptionHolder[0] = e; + }); + IngestDocument document = documentHolder[0]; + Exception exception = exceptionHolder[0]; + assertNull(document); + assertNotNull(exception); + assertThat(exception.getMessage(), containsString("Too many nested pipelines")); + // We expect that the last processor was never called: + assertThat(countCallsProcessor.getTotalCount(), equalTo(0)); + } + public void testActualPipelineProcessorRepeatedInvocation() throws Exception { String pipelineId = "pipeline1"; IngestService ingestService = createIngestService(); From 6780739ffdb38130a4464ab63db15b501965f8a3 Mon Sep 17 00:00:00 2001 From: Kyungeun Kim Date: Wed, 14 Feb 2024 12:25:51 +0900 Subject: [PATCH 17/78] x-pack/plugin/apm-data: add transaction.profiler_stack_trace_ids field to apm plugin (#105223) * feat: add transaction.profiler_stack_trace_ids field to apm plugin * docs: add changelog file * fix: add counted-keyword module --- docs/changelog/105223.yaml | 5 +++++ x-pack/plugin/apm-data/build.gradle | 1 + .../resources/component-templates/traces-apm@mappings.yaml | 2 ++ .../org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java | 1 + 4 files changed, 9 insertions(+) create mode 100644 docs/changelog/105223.yaml diff --git a/docs/changelog/105223.yaml b/docs/changelog/105223.yaml new file mode 100644 index 0000000000000..e2a95fcd6ba48 --- /dev/null +++ b/docs/changelog/105223.yaml @@ -0,0 +1,5 @@ +pr: 105223 +summary: "x-pack/plugin/apm-data: Add a new field transaction.profiler_stack_trace_ids to traces-apm@mappings.yaml" +area: Data streams +type: enhancement +issues: [] diff --git a/x-pack/plugin/apm-data/build.gradle b/x-pack/plugin/apm-data/build.gradle index 0bcd66f18a907..cbd843d227ff4 100644 --- a/x-pack/plugin/apm-data/build.gradle +++ b/x-pack/plugin/apm-data/build.gradle @@ -30,6 +30,7 @@ dependencies { clusterModules project(xpackModule('ilm')) clusterModules project(xpackModule('mapper-aggregate-metric')) clusterModules project(xpackModule('mapper-constant-keyword')) + clusterModules project(xpackModule('mapper-counted-keyword')) clusterModules project(xpackModule('stack')) clusterModules project(xpackModule('wildcard')) } diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml index eb2da017d97b7..780fce37e1d40 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml @@ -51,3 +51,5 @@ template: type: scaled_float scaling_factor: 1000 index: false + transaction.profiler_stack_trace_ids: + type: counted_keyword diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java b/x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java index 3941f9a2e6788..1ed3892e2f8f4 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java +++ b/x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java @@ -20,6 +20,7 @@ public class APMYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("constant-keyword") + .module("counted-keyword") .module("data-streams") .module("ingest-common") .module("ingest-geoip") From 67bf5f3d28bf8d99de327b1c0224822874ab781e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 14 Feb 2024 08:10:25 +0100 Subject: [PATCH 18/78] Improve test coverage for index shrinking a tsdb index. (#105459) --- .../TSDBPassthroughIndexingIT.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index b58aa201d077f..5d84baa5f6ea4 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -8,12 +8,19 @@ package org.elasticsearch.datastreams; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -151,6 +158,10 @@ public void testIndexingGettingAndSearching() throws Exception { assertHitCount(searchResponse, 1); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo(id)); }); + var deleteResponse = client().delete(new DeleteRequest(index, id)).actionGet(); + assertThat(deleteResponse.getIndex(), equalTo(index)); + assertThat(deleteResponse.getId(), equalTo(id)); + assertThat(deleteResponse.getResult(), equalTo(DocWriteResponse.Result.DELETED)); time = time.plusMillis(1); } @@ -187,6 +198,89 @@ public void testIndexingGettingAndSearching() throws Exception { ); } + public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { + String dataStreamName = "k8s"; + var templateSettings = Settings.builder() + .put("index.mode", "time_series") + .put("index.number_of_shards", 8) + .put("index.number_of_replicas", 0); + + var request = new TransportPutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of("k8s*")) + .template(new Template(templateSettings.build(), new CompressedXContent(MAPPING_TEMPLATE), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + Instant time = Instant.now(); + int numBulkItems = randomIntBetween(16, 128); + var bulkRequest = new BulkRequest(dataStreamName); + for (int i = 0; i < numBulkItems; i++) { + var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); + indexRequest.source( + DOC.replace("$time", formatInstant(time)) + .replace("$uid", randomUUID()) + .replace("$name", randomAlphaOfLength(4)) + .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), + XContentType.JSON + ); + bulkRequest.add(indexRequest); + time = time.plusMillis(1); + } + + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + var bulkResponse = client().bulk(bulkRequest).actionGet(); + for (var itemResponse : bulkResponse) { + String id = itemResponse.getId(); + String index = itemResponse.getIndex(); + var getResponse = client().get(new GetRequest(index, id)).actionGet(); + assertThat(getResponse.isExists(), is(true)); + + var searchRequest = new SearchRequest(index); + searchRequest.source(new SearchSourceBuilder().query(new TermQueryBuilder("_id", id))); + assertResponse(client().search(searchRequest), searchResponse -> { + assertHitCount(searchResponse, 1); + assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo(id)); + }); + } + + var rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet(); + assertThat(rolloverResponse.isRolledOver(), is(true)); + String sourceIndex = rolloverResponse.getOldIndex(); + + var updateSettingsResponse = client().admin() + .indices() + .updateSettings(new UpdateSettingsRequest(sourceIndex).settings(Settings.builder().put("index.blocks.write", true))) + .actionGet(); + assertThat(updateSettingsResponse.isAcknowledged(), is(true)); + + String shrunkenTarget = "k8s-shrunken"; + var shrinkIndexResponse = client().admin() + .indices() + .prepareResizeIndex(sourceIndex, shrunkenTarget) + .setResizeType(ResizeType.SHRINK) + .setSettings(indexSettings(2, 0).build()) + .get(); + assertThat(shrinkIndexResponse.isAcknowledged(), is(true)); + assertThat(shrinkIndexResponse.index(), equalTo(shrunkenTarget)); + + for (var itemResponse : bulkResponse) { + String id = itemResponse.getId(); + var getResponse = client().get(new GetRequest(shrunkenTarget, id)).actionGet(); + assertThat(getResponse.isExists(), is(true)); + + var searchRequest = new SearchRequest(shrunkenTarget); + searchRequest.source(new SearchSourceBuilder().query(new TermQueryBuilder("_id", id))); + assertResponse(client().search(searchRequest), searchResponse -> { + assertHitCount(searchResponse, 1); + assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo(id)); + }); + } + } + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } From a0ed9ba73e5eb0ab12f3ceab0893022393f3a214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 14 Feb 2024 08:11:32 +0100 Subject: [PATCH 19/78] Move `MlConfigVersion.getMinMlConfigVersion` call under try-catch block (#105391) --- docs/changelog/105391.yaml | 5 +++++ .../xpack/ml/inference/ingest/InferenceProcessor.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/105391.yaml diff --git a/docs/changelog/105391.yaml b/docs/changelog/105391.yaml new file mode 100644 index 0000000000000..6b9b39c00a150 --- /dev/null +++ b/docs/changelog/105391.yaml @@ -0,0 +1,5 @@ +pr: 105391 +summary: Catch all the potential exceptions in the ingest processor code +area: Machine Learning +type: bug +issues: [] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java index 470605dcb2d9c..6b28a9aef9f48 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java @@ -381,8 +381,8 @@ public Factory(Client client, ClusterService clusterService, Settings settings, @Override public void accept(ClusterState state) { - minNodeVersion = MlConfigVersion.getMinMlConfigVersion(state.nodes()); try { + minNodeVersion = MlConfigVersion.getMinMlConfigVersion(state.nodes()); currentInferenceProcessors = InferenceProcessorInfoExtractor.countInferenceProcessors(state); } catch (Exception ex) { // We cannot throw any exception here. It might break other pipelines. From 741c6327ca741c7f6237592d29dd267b91e65b1c Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 14 Feb 2024 10:09:56 +0200 Subject: [PATCH 20/78] [Health Monitoring] Stop the periodic health logger when es is stopping (#105272) We see errors that we believe this is happening because `es` is already stopping but the periodic health logger keeps querying the the health API. Since the `es` stopping we believe it makes sense to also stop the periodic health logger. Furthermore, we make the close method more respectful to the execution of the periodic health logger which will wait for the last run to finish if it's still in progress. This PR makes the `HealthPeriodicLogger` lifecycle aware and uses a semaphore to block the `close()` method. --- docs/changelog/105272.yaml | 5 + .../health/HealthPeriodicLogger.java | 154 +++++-- .../java/org/elasticsearch/node/Node.java | 3 + .../health/HealthPeriodicLoggerTests.java | 432 +++++++++++++----- 4 files changed, 447 insertions(+), 147 deletions(-) create mode 100644 docs/changelog/105272.yaml diff --git a/docs/changelog/105272.yaml b/docs/changelog/105272.yaml new file mode 100644 index 0000000000000..1032a17fc10f8 --- /dev/null +++ b/docs/changelog/105272.yaml @@ -0,0 +1,5 @@ +pr: 105272 +summary: "Stop the periodic health logger when es is stopping" +area: Health +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java b/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java index e067150c911df..9d3df9a6b01d2 100644 --- a/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java +++ b/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java @@ -17,7 +17,9 @@ import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.logging.ESLogMessage; import org.elasticsearch.common.scheduler.SchedulerEngine; import org.elasticsearch.common.scheduler.TimeValueSchedule; @@ -31,7 +33,7 @@ import org.elasticsearch.telemetry.metric.LongGaugeMetric; import org.elasticsearch.telemetry.metric.MeterRegistry; -import java.io.Closeable; +import java.io.IOException; import java.time.Clock; import java.util.EnumSet; import java.util.HashMap; @@ -39,7 +41,8 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -47,9 +50,13 @@ import static org.elasticsearch.health.HealthStatus.RED; /** - * This class periodically logs the results of the Health API to the standard Elasticsearch server log file. + * This class periodically logs the results of the Health API to the standard Elasticsearch server log file. It a lifecycle + * aware component because it health depends on other lifecycle aware components. This means: + * - We do not schedule any jobs until the lifecycle state is STARTED + * - When the lifecycle state becomes STOPPED, do not schedule any more runs, but we do let the current one finish + * - When the lifecycle state becomes CLOSED, we will interrupt the current run as well. */ -public class HealthPeriodicLogger implements ClusterStateListener, Closeable, SchedulerEngine.Listener { +public class HealthPeriodicLogger extends AbstractLifecycleComponent implements ClusterStateListener, SchedulerEngine.Listener { public static final String HEALTH_FIELD_PREFIX = "elasticsearch.health"; public static final String MESSAGE_FIELD = "message"; @@ -120,10 +127,10 @@ static OutputMode parseOutputMode(String value) { private final HealthService healthService; private final Clock clock; - // default visibility for testing purposes - volatile boolean isHealthNode = false; + private volatile boolean isHealthNode = false; - private final AtomicBoolean currentlyRunning = new AtomicBoolean(false); + // This semaphore is used to ensure only one schedule is currently running and to wait for this run to finish before closing + private final Semaphore currentlyRunning = new Semaphore(1, true); private final SetOnce scheduler = new SetOnce<>(); private volatile TimeValue pollInterval; @@ -169,7 +176,7 @@ static HealthPeriodicLogger create( BiConsumer metricWriter, Consumer logWriter ) { - HealthPeriodicLogger logger = new HealthPeriodicLogger( + HealthPeriodicLogger healthLogger = new HealthPeriodicLogger( settings, clusterService, client, @@ -178,8 +185,8 @@ static HealthPeriodicLogger create( metricWriter, logWriter ); - logger.registerListeners(); - return logger; + healthLogger.registerListeners(); + return healthLogger; } private HealthPeriodicLogger( @@ -217,6 +224,17 @@ private void registerListeners() { clusterService.getClusterSettings().addSettingsUpdateConsumer(ENABLED_SETTING, this::enable); clusterService.getClusterSettings().addSettingsUpdateConsumer(POLL_INTERVAL_SETTING, this::updatePollInterval); clusterService.getClusterSettings().addSettingsUpdateConsumer(OUTPUT_MODE_SETTING, this::updateOutputModes); + this.addLifecycleListener(new LifecycleListener() { + @Override + public void afterStart() { + maybeScheduleJob(); + } + + @Override + public void afterStop() { + maybeCancelJob(); + } + }); } @Override @@ -246,10 +264,35 @@ public void clusterChanged(ClusterChangedEvent event) { } @Override - public void close() { - SchedulerEngine engine = scheduler.get(); - if (engine != null) { - engine.stop(); + protected void doStart() { + logger.debug("Periodic health logger is starting."); + } + + /** + * Stopping means that the periodic health logger will not schedule any more runs. If the logger is currently running it will + * let this run finish, but it will cancel any future scheduling, and it will deregister the cluster state listener. + */ + @Override + protected void doStop() { + clusterService.removeListener(this); + logger.debug("Periodic health logger is stopping."); + } + + @Override + protected void doClose() throws IOException { + logger.debug("Periodic health logger is closing."); + try { + // The health API is expected to be a quick call, so we do not need a very long timeout + if (currentlyRunning.tryAcquire(2, TimeUnit.SECONDS)) { + logger.debug("Periodic health logger's last run has successfully finished."); + } + } catch (InterruptedException e) { + logger.warn("Error while waiting for the last run of the periodic health logger to finish.", e); + } finally { + SchedulerEngine engine = scheduler.get(); + if (engine != null) { + engine.stop(); + } } } @@ -260,20 +303,29 @@ public void triggered(SchedulerEngine.Event event) { } } - // default visibility for testing purposes - void tryToLogHealth() { - if (this.currentlyRunning.compareAndExchange(false, true) == false) { - RunOnce release = new RunOnce(() -> currentlyRunning.set(false)); - try { - ActionListener> listenerWithRelease = ActionListener.runAfter(resultsListener, release); - this.healthService.getHealth(this.client, null, false, 0, listenerWithRelease); - } catch (Exception e) { - logger.warn(() -> "The health periodic logger encountered an error.", e); - // In case of an exception before the listener was wired, we can release the flag here, and we feel safe - // that it will not release it again because this can only be run once. - release.run(); + // default visibility and returns true if the semaphore was acquired, used only for testing + boolean tryToLogHealth() { + try { + // We only try to run this because we do not want to pile up the executions. + if (currentlyRunning.tryAcquire(0, TimeUnit.SECONDS)) { + RunOnce release = new RunOnce(currentlyRunning::release); + try { + ActionListener> listenerWithRelease = ActionListener.runAfter(resultsListener, release); + this.healthService.getHealth(this.client, null, false, 0, listenerWithRelease); + } catch (Exception e) { + // In case of an exception before the listener was wired, we can release the flag here, and we feel safe + // that it will not release it again because this can only be run once. + release.run(); + logger.warn(() -> "The health periodic logger encountered an error.", e); + } + return true; + } else { + logger.debug("Skipping this run because it's already in progress."); } + } catch (InterruptedException e) { + logger.debug("Periodic health logger run was interrupted.", e); } + return false; } // default visibility for testing purposes @@ -417,11 +469,11 @@ private void maybeScheduleJob() { return; } - // don't schedule the job if the node is shutting down - if (isClusterServiceStoppedOrClosed()) { + // don't schedule the job if the node is not started yet, or it's shutting down + if (isStarted() == false) { logger.trace( - "Skipping scheduling a health periodic logger job due to the cluster lifecycle state being: [{}] ", - clusterService.lifecycleState() + "Skipping scheduling a health periodic logger job due to the health logger lifecycle state being: [{}] ", + this.lifecycleState() ); return; } @@ -447,7 +499,8 @@ private void maybeCancelJob() { private void enable(boolean enabled) { this.enabled = enabled; - if (enabled) { + // After the health logger is stopped we do not want to reschedule it + if (enabled & isStoppedOrClosed() == false) { clusterService.addListener(this); maybeScheduleJob(); } else { @@ -458,11 +511,42 @@ private void enable(boolean enabled) { private void updatePollInterval(TimeValue newInterval) { this.pollInterval = newInterval; - maybeScheduleJob(); + // After the health logger is stopped we do not want to reschedule it + if (isStoppedOrClosed() == false) { + maybeScheduleJob(); + } + } + + private boolean isStarted() { + return lifecycleState() == Lifecycle.State.STARTED; + } + + private boolean isStoppedOrClosed() { + return lifecycleState() == Lifecycle.State.STOPPED || lifecycleState() == Lifecycle.State.CLOSED; + } + + // Visible for testing + TimeValue getPollInterval() { + return pollInterval; + } + + // Visible for testing + boolean isHealthNode() { + return isHealthNode; + } + + // Visible for testing + boolean enabled() { + return enabled; + } + + // Visible for testing + boolean currentlyRunning() { + return currentlyRunning.availablePermits() == 0; } - private boolean isClusterServiceStoppedOrClosed() { - final Lifecycle.State state = clusterService.lifecycleState(); - return state == Lifecycle.State.STOPPED || state == Lifecycle.State.CLOSED; + // Visible for testing + boolean waitingToFinishCurrentRun() { + return currentlyRunning.hasQueuedThreads(); } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 6da1dbc3e5c52..165c5f6524104 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -432,6 +432,7 @@ public void onTimeout(TimeValue timeout) { } injector.getInstance(NodeMetrics.class).start(); + injector.getInstance(HealthPeriodicLogger.class).start(); logger.info("started {}", transportService.getLocalNode()); @@ -456,6 +457,8 @@ private void stop() { if (ReadinessService.enabled(environment)) { stopIfStarted(ReadinessService.class); } + // We stop the health periodic logger first since certain checks won't be possible anyway + stopIfStarted(HealthPeriodicLogger.class); stopIfStarted(FileSettingsService.class); injector.getInstance(ResourceWatcherService.class).close(); stopIfStarted(HttpServerTransport.class); diff --git a/server/src/test/java/org/elasticsearch/health/HealthPeriodicLoggerTests.java b/server/src/test/java/org/elasticsearch/health/HealthPeriodicLoggerTests.java index c52b0f594ed71..02bd0852f50c4 100644 --- a/server/src/test/java/org/elasticsearch/health/HealthPeriodicLoggerTests.java +++ b/server/src/test/java/org/elasticsearch/health/HealthPeriodicLoggerTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.logging.ESLogMessage; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.scheduler.SchedulerEngine; @@ -42,8 +43,11 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -60,7 +64,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; public class HealthPeriodicLoggerTests extends ESTestCase { private ThreadPool threadPool; @@ -105,7 +108,13 @@ public void setupServices() { public void cleanup() { clusterService.close(); if (testHealthPeriodicLogger != null) { - testHealthPeriodicLogger.close(); + if (testHealthPeriodicLogger.lifecycleState() == Lifecycle.State.STARTED) { + testHealthPeriodicLogger.stop(); + } + if (testHealthPeriodicLogger.lifecycleState() == Lifecycle.State.INITIALIZED + || testHealthPeriodicLogger.lifecycleState() == Lifecycle.State.STOPPED) { + testHealthPeriodicLogger.close(); + } } threadPool.shutdownNow(); } @@ -157,32 +166,42 @@ public void testConvertToLoggedFields() { public void testHealthNodeIsSelected() { HealthService testHealthService = this.getMockedHealthService(); - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(clusterService, testHealthService, true); + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(clusterService, testHealthService, randomBoolean()); // test that it knows that it's not initially the health node - assertFalse(testHealthPeriodicLogger.isHealthNode); + assertFalse(testHealthPeriodicLogger.isHealthNode()); // trigger a cluster change and recheck testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); - assertTrue(testHealthPeriodicLogger.isHealthNode); + assertTrue(testHealthPeriodicLogger.isHealthNode()); } - public void testJobScheduling() { + public void testJobScheduling() throws Exception { HealthService testHealthService = this.getMockedHealthService(); - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(clusterService, testHealthService, true); + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(clusterService, testHealthService, false); testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); - assertTrue(testHealthPeriodicLogger.isHealthNode); - - SchedulerEngine scheduler = testHealthPeriodicLogger.getScheduler(); - assertNotNull(scheduler); - assertTrue(scheduler.scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME)); + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); + assertTrue("health logger should be enabled", testHealthPeriodicLogger.enabled()); + // Even if this is the health node, we do not schedule a job because the service is not started yet + assertNull(testHealthPeriodicLogger.getScheduler()); + // Starting the service should schedule a try to schedule a run + testHealthPeriodicLogger.start(); + AtomicReference scheduler = new AtomicReference<>(); + assertBusy(() -> { + var s = testHealthPeriodicLogger.getScheduler(); + assertNotNull(s); + scheduler.set(s); + }); + assertTrue(scheduler.get().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME)); + + // Changing the health node should cancel the run ClusterState noHealthNode = ClusterStateCreationUtils.state(node2, node1, new DiscoveryNode[] { node1, node2 }); testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", noHealthNode, stateWithLocalHealthNode)); - assertFalse(testHealthPeriodicLogger.isHealthNode); - assertFalse(scheduler.scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME)); + assertFalse(testHealthPeriodicLogger.isHealthNode()); + assertFalse(scheduler.get().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME)); } public void testEnabled() { @@ -190,11 +209,12 @@ public void testEnabled() { testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(clusterService, testHealthService, true); testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); - assertTrue(testHealthPeriodicLogger.isHealthNode); + verifyLoggerIsReadyToRun(testHealthPeriodicLogger); // disable it and then verify that the job is gone { this.clusterSettings.applySettings(Settings.builder().put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), false).build()); + assertFalse(testHealthPeriodicLogger.enabled()); assertFalse( testHealthPeriodicLogger.getScheduler().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME) ); @@ -203,60 +223,119 @@ public void testEnabled() { // enable it and then verify that the job is created { this.clusterSettings.applySettings(Settings.builder().put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true).build()); + assertTrue(testHealthPeriodicLogger.enabled()); assertTrue( testHealthPeriodicLogger.getScheduler().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME) ); } + // ensure the job is not recreated during enabling if the service has stopped + { + testHealthPeriodicLogger.stop(); + assertFalse( + testHealthPeriodicLogger.getScheduler().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME) + ); + this.clusterSettings.applySettings(Settings.builder().put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true).build()); + assertTrue(testHealthPeriodicLogger.enabled()); + assertFalse( + testHealthPeriodicLogger.getScheduler().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME) + ); + } } public void testUpdatePollInterval() { HealthService testHealthService = this.getMockedHealthService(); - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(clusterService, testHealthService, false); - assertNull(testHealthPeriodicLogger.getScheduler()); - testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); - assertTrue(testHealthPeriodicLogger.isHealthNode); - - // verify that updating the polling interval doesn't schedule the job + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); + assertTrue("health logger should be enabled", testHealthPeriodicLogger.enabled()); + // Ensure updating the poll interval won't trigger a job when service not started { + TimeValue pollInterval = TimeValue.timeValueSeconds(randomIntBetween(15, 59)); this.clusterSettings.applySettings( - Settings.builder().put(HealthPeriodicLogger.POLL_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(15)).build() + Settings.builder() + .put(HealthPeriodicLogger.POLL_INTERVAL_SETTING.getKey(), pollInterval) + // Since the default value of enabled is false, if we do not set it here it disable it + .put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true) + .build() ); + assertTrue("health logger should be enabled", testHealthPeriodicLogger.enabled()); + assertEquals(pollInterval, testHealthPeriodicLogger.getPollInterval()); assertNull(testHealthPeriodicLogger.getScheduler()); } - } - public void testTriggeredJobCallsTryToLogHealth() throws Exception { - AtomicBoolean listenerCalled = new AtomicBoolean(false); - AtomicBoolean failureCalled = new AtomicBoolean(false); - ActionListener> testListener = new ActionListener<>() { - @Override - public void onResponse(List indicatorResults) { - listenerCalled.set(true); - } + testHealthPeriodicLogger.start(); + // Start the service and check it's scheduled + { + TimeValue pollInterval = TimeValue.timeValueSeconds(randomIntBetween(15, 59)); + this.clusterSettings.applySettings( + Settings.builder() + .put(HealthPeriodicLogger.POLL_INTERVAL_SETTING.getKey(), pollInterval) + // Since the default value of enabled is false, if we do not set it here it disable it + .put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true) + .build() + ); + assertEquals(pollInterval, testHealthPeriodicLogger.getPollInterval()); + verifyLoggerIsReadyToRun(testHealthPeriodicLogger); + assertNotNull(testHealthPeriodicLogger.getScheduler()); + assertTrue( + testHealthPeriodicLogger.getScheduler().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME) + ); + } - @Override - public void onFailure(Exception e) { - failureCalled.set(true); - } - }; - HealthService testHealthService = this.getMockedHealthService(); + // Poll interval doesn't schedule a job when disabled + { + TimeValue pollInterval = TimeValue.timeValueSeconds(randomIntBetween(15, 59)); + this.clusterSettings.applySettings( + Settings.builder() + .put(HealthPeriodicLogger.POLL_INTERVAL_SETTING.getKey(), pollInterval) + .put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), false) + .build() + ); + assertFalse(testHealthPeriodicLogger.enabled()); + assertEquals(pollInterval, testHealthPeriodicLogger.getPollInterval()); + assertFalse( + testHealthPeriodicLogger.getScheduler().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME) + ); + // Re-enable + this.clusterSettings.applySettings( + Settings.builder().put(HealthPeriodicLogger.POLL_INTERVAL_SETTING.getKey(), pollInterval).build() + ); + } - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); + testHealthPeriodicLogger.stop(); + // verify that updating the polling interval doesn't schedule the job if it's stopped + { + this.clusterSettings.applySettings( + Settings.builder() + .put(HealthPeriodicLogger.POLL_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(30)) + .put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true) + .build() + ); + assertTrue("health logger should be enabled", testHealthPeriodicLogger.enabled()); + assertFalse( + testHealthPeriodicLogger.getScheduler().scheduledJobIds().contains(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME) + ); + } + } - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; + public void testTriggeredJobCallsTryToLogHealth() throws Exception { + AtomicBoolean calledGetHealth = new AtomicBoolean(); + HealthService testHealthService = this.getMockedHealthService(); doAnswer(invocation -> { - testListener.onResponse(getTestIndicatorResults()); + ActionListener> listener = invocation.getArgument(4); + assertNotNull(listener); + calledGetHealth.set(true); + listener.onResponse(getTestIndicatorResults()); return null; - }).when(spyHealthPeriodicLogger).tryToLogHealth(); + }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); - SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); - spyHealthPeriodicLogger.triggered(event); + verifyLoggerIsReadyToRun(testHealthPeriodicLogger); - assertBusy(() -> assertFalse(failureCalled.get())); - assertBusy(() -> assertTrue(listenerCalled.get())); + SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); + testHealthPeriodicLogger.triggered(event); + assertBusy(() -> assertTrue(calledGetHealth.get())); } public void testResultFailureHandling() throws Exception { @@ -265,9 +344,8 @@ public void testResultFailureHandling() throws Exception { HealthService testHealthService = this.getMockedHealthService(); testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); - - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); @@ -279,7 +357,7 @@ public void testResultFailureHandling() throws Exception { getHealthCalled.incrementAndGet(); return null; }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); - spyHealthPeriodicLogger.triggered(event); + testHealthPeriodicLogger.triggered(event); assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(1))); } @@ -291,7 +369,7 @@ public void testResultFailureHandling() throws Exception { getHealthCalled.incrementAndGet(); return null; }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); - spyHealthPeriodicLogger.triggered(event); + testHealthPeriodicLogger.triggered(event); assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(2))); } } @@ -299,31 +377,41 @@ public void testResultFailureHandling() throws Exception { public void testTryToLogHealthConcurrencyControlWithResults() throws Exception { AtomicInteger getHealthCalled = new AtomicInteger(0); + CountDownLatch waitForSecondRun = new CountDownLatch(1); + CountDownLatch waitForRelease = new CountDownLatch(1); HealthService testHealthService = this.getMockedHealthService(); doAnswer(invocation -> { // get and call the results listener provided to getHealth ActionListener> listener = invocation.getArgument(4); - listener.onResponse(getTestIndicatorResults()); getHealthCalled.incrementAndGet(); + waitForSecondRun.await(); + listener.onResponse(getTestIndicatorResults()); + waitForRelease.countDown(); return null; }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); - - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + verifyLoggerIsReadyToRun(testHealthPeriodicLogger); SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); // run it once, verify getHealth is called { - spyHealthPeriodicLogger.triggered(event); + Thread logHealthThread = new Thread(() -> testHealthPeriodicLogger.triggered(event)); + logHealthThread.start(); + // We wait to verify that the triggered even is in progress, then we block, so it will rename in progress assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(1))); + // We try to log again while it's in progress, we expect this run to be skipped + assertFalse(testHealthPeriodicLogger.tryToLogHealth()); + // Unblock the first execution + waitForSecondRun.countDown(); } // run it again, verify getHealth is called, because we are calling the results listener { - spyHealthPeriodicLogger.triggered(event); + waitForRelease.await(); + testHealthPeriodicLogger.triggered(event); assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(2))); } } @@ -331,36 +419,51 @@ public void testTryToLogHealthConcurrencyControlWithResults() throws Exception { public void testTryToLogHealthConcurrencyControl() throws Exception { AtomicInteger getHealthCalled = new AtomicInteger(0); + CountDownLatch waitForSecondRun = new CountDownLatch(1); + CountDownLatch waitForRelease = new CountDownLatch(1); + HealthService testHealthService = this.getMockedHealthService(); doAnswer(invocation -> { - // get but do not call the provided listener + // get but do not call the provided listener immediately ActionListener> listener = invocation.getArgument(4); assertNotNull(listener); // note that we received the getHealth call getHealthCalled.incrementAndGet(); + + // wait for the next run that should be skipped + waitForSecondRun.await(); + // we can continue now + listener.onResponse(getTestIndicatorResults()); + waitForRelease.countDown(); return null; }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); - - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, false); + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); - // call it once, and verify that getHealth is called + // call it and verify that getHealth is called { - spyHealthPeriodicLogger.triggered(event); + Thread logHealthThread = new Thread(() -> testHealthPeriodicLogger.triggered(event)); + logHealthThread.start(); assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(1))); } - // trigger it again, and verify that getHealth is not called - // it's not called because the results listener was never called by getHealth - // this is simulating a double invocation of getHealth, due perhaps to a lengthy getHealth call + // run it again, verify that it's skipped because the other one is in progress { - spyHealthPeriodicLogger.triggered(event); - assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(1))); + assertFalse(testHealthPeriodicLogger.tryToLogHealth()); + // Unblock the first execution + waitForSecondRun.countDown(); + } + + // run it again, verify getHealth is called, because we are calling the results listener + { + waitForRelease.await(); + testHealthPeriodicLogger.triggered(event); + assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(2))); } } @@ -369,10 +472,9 @@ public void testTryToLogHealthConcurrencyControlWithException() throws Exception HealthService testHealthService = this.getMockedHealthService(); - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); - - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, false); + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); @@ -380,7 +482,7 @@ public void testTryToLogHealthConcurrencyControlWithException() throws Exception { doThrow(new ResourceNotFoundException("No preflight indicators")).when(testHealthService) .getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); - spyHealthPeriodicLogger.triggered(event); + testHealthPeriodicLogger.triggered(event); assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(0))); } @@ -392,11 +494,102 @@ public void testTryToLogHealthConcurrencyControlWithException() throws Exception getHealthCalled.incrementAndGet(); return null; }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); - spyHealthPeriodicLogger.triggered(event); + testHealthPeriodicLogger.triggered(event); assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(1))); } } + public void testClosingWhenRunInProgress() throws Exception { + // Check that closing will still happen even if the run doesn't finish + { + AtomicInteger getHealthCalled = new AtomicInteger(0); + + HealthService testHealthService = this.getMockedHealthService(); + doAnswer(invocation -> { + ActionListener> listener = invocation.getArgument(4); + assertNotNull(listener); + + // note that we received the getHealth call + getHealthCalled.incrementAndGet(); + return null; + }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); + + HealthPeriodicLogger healthLoggerThatWillNotFinish = createAndInitHealthPeriodicLogger( + this.clusterService, + testHealthService, + true + ); + healthLoggerThatWillNotFinish.clusterChanged( + new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE) + ); + assertTrue("local node should be the health node", healthLoggerThatWillNotFinish.isHealthNode()); + assertTrue("health logger should be enabled", healthLoggerThatWillNotFinish.enabled()); + + SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); + + // call it and verify that it's in progress + { + healthLoggerThatWillNotFinish.triggered(event); + assertBusy(() -> assertThat(getHealthCalled.get(), equalTo(1))); + } + healthLoggerThatWillNotFinish.stop(); + assertEquals(Lifecycle.State.STOPPED, healthLoggerThatWillNotFinish.lifecycleState()); + // Close and wait out the timeout + healthLoggerThatWillNotFinish.close(); + assertBusy(() -> assertEquals(Lifecycle.State.CLOSED, healthLoggerThatWillNotFinish.lifecycleState()), 5, TimeUnit.SECONDS); + } + + // Ensure it will wait until it finishes before it closes + { + AtomicInteger getHealthCalled = new AtomicInteger(0); + + CountDownLatch waitForCloseToBeTriggered = new CountDownLatch(1); + CountDownLatch waitForRelease = new CountDownLatch(1); + + HealthService testHealthService = this.getMockedHealthService(); + doAnswer(invocation -> { + // get but do not call the provided listener immediately + ActionListener> listener = invocation.getArgument(4); + assertNotNull(listener); + + // note that we received the getHealth call + getHealthCalled.incrementAndGet(); + + // wait for the close signal + waitForCloseToBeTriggered.await(); + // we can continue now + listener.onResponse(getTestIndicatorResults()); + waitForRelease.countDown(); + return null; + }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); + + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + verifyLoggerIsReadyToRun(testHealthPeriodicLogger); + + SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); + + // call it and verify that getHealth is called + { + Thread logHealthThread = new Thread(() -> testHealthPeriodicLogger.triggered(event)); + logHealthThread.start(); + assertBusy(() -> assertTrue(testHealthPeriodicLogger.currentlyRunning())); + } + + // stop and close it + { + testHealthPeriodicLogger.stop(); + assertEquals(Lifecycle.State.STOPPED, testHealthPeriodicLogger.lifecycleState()); + assertTrue(testHealthPeriodicLogger.currentlyRunning()); + Thread closeHealthLogger = new Thread(() -> testHealthPeriodicLogger.close()); + closeHealthLogger.start(); + assertBusy(() -> assertTrue(testHealthPeriodicLogger.waitingToFinishCurrentRun())); + waitForCloseToBeTriggered.countDown(); + assertBusy(() -> assertEquals(Lifecycle.State.CLOSED, testHealthPeriodicLogger.lifecycleState())); + } + } + } + public void testLoggingHappens() { MockLogAppender mockAppender = new MockLogAppender(); mockAppender.start(); @@ -436,8 +629,13 @@ public void testLoggingHappens() { Loggers.addAppender(periodicLoggerLogger, mockAppender); HealthService testHealthService = this.getMockedHealthService(); - - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); + doAnswer(invocation -> { + ActionListener> listener = invocation.getArgument(4); + assertNotNull(listener); + listener.onResponse(getTestIndicatorResults()); + return null; + }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, false); // switch to Log only mode this.clusterSettings.applySettings( @@ -446,16 +644,11 @@ public void testLoggingHappens() { .put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true) .build() ); - - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; - doAnswer(invocation -> { - spyHealthPeriodicLogger.resultsListener.onResponse(getTestIndicatorResults()); - return null; - }).when(spyHealthPeriodicLogger).tryToLogHealth(); + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); - spyHealthPeriodicLogger.triggered(event); + testHealthPeriodicLogger.triggered(event); try { mockAppender.assertAllExpectationsMatched(); @@ -496,8 +689,13 @@ public void testOutputModeNoLogging() { Loggers.addAppender(periodicLoggerLogger, mockAppender); HealthService testHealthService = this.getMockedHealthService(); - - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true); + doAnswer(invocation -> { + ActionListener> listener = invocation.getArgument(4); + assertNotNull(listener); + listener.onResponse(getTestIndicatorResults()); + return null; + }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, false); // switch to Metrics only mode this.clusterSettings.applySettings( @@ -506,16 +704,11 @@ public void testOutputModeNoLogging() { .put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true) .build() ); - - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; - doAnswer(invocation -> { - spyHealthPeriodicLogger.resultsListener.onResponse(getTestIndicatorResults()); - return null; - }).when(spyHealthPeriodicLogger).tryToLogHealth(); + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); - spyHealthPeriodicLogger.triggered(event); + testHealthPeriodicLogger.triggered(event); try { mockAppender.assertAllExpectationsMatched(); @@ -531,9 +724,21 @@ public void testMetricsMode() { BiConsumer metricWriter = (metric, value) -> metrics.add(value); Consumer logWriter = msg -> logs.add(msg.asString()); - + List results = getTestIndicatorResultsWithRed(); HealthService testHealthService = this.getMockedHealthService(); - testHealthPeriodicLogger = createAndInitHealthPeriodicLogger(this.clusterService, testHealthService, true, metricWriter, logWriter); + doAnswer(invocation -> { + ActionListener> listener = invocation.getArgument(4); + assertNotNull(listener); + listener.onResponse(results); + return null; + }).when(testHealthService).getHealth(any(), isNull(), anyBoolean(), anyInt(), any()); + testHealthPeriodicLogger = createAndInitHealthPeriodicLogger( + this.clusterService, + testHealthService, + false, + metricWriter, + logWriter + ); // switch to Metrics only mode this.clusterSettings.applySettings( @@ -542,25 +747,24 @@ public void testMetricsMode() { .put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true) .build() ); - - HealthPeriodicLogger spyHealthPeriodicLogger = spy(testHealthPeriodicLogger); - spyHealthPeriodicLogger.isHealthNode = true; - List results = getTestIndicatorResultsWithRed(); - - doAnswer(invocation -> { - spyHealthPeriodicLogger.resultsListener.onResponse(results); - return null; - }).when(spyHealthPeriodicLogger).tryToLogHealth(); + testHealthPeriodicLogger.clusterChanged(new ClusterChangedEvent("test", stateWithLocalHealthNode, ClusterState.EMPTY_STATE)); + assertTrue("local node should be the health node", testHealthPeriodicLogger.isHealthNode()); assertEquals(0, metrics.size()); SchedulerEngine.Event event = new SchedulerEngine.Event(HealthPeriodicLogger.HEALTH_PERIODIC_LOGGER_JOB_NAME, 0, 0); - spyHealthPeriodicLogger.triggered(event); + testHealthPeriodicLogger.triggered(event); assertEquals(0, logs.size()); assertEquals(4, metrics.size()); } + private void verifyLoggerIsReadyToRun(HealthPeriodicLogger healthPeriodicLogger) { + assertTrue("local node should be the health node", healthPeriodicLogger.isHealthNode()); + assertTrue("health logger should be enabled", healthPeriodicLogger.enabled()); + assertEquals("health logger is started", Lifecycle.State.STARTED, healthPeriodicLogger.lifecycleState()); + } + private List getTestIndicatorResults() { var networkLatency = new HealthIndicatorResult("master_is_stable", GREEN, null, null, null, null); var slowTasks = new HealthIndicatorResult("disk", YELLOW, null, null, null, null); @@ -592,15 +796,15 @@ private String makeHealthStatusString(String key) { private HealthPeriodicLogger createAndInitHealthPeriodicLogger( ClusterService clusterService, HealthService testHealthService, - boolean enabled + boolean started ) { - return createAndInitHealthPeriodicLogger(clusterService, testHealthService, enabled, null, null); + return createAndInitHealthPeriodicLogger(clusterService, testHealthService, started, null, null); } private HealthPeriodicLogger createAndInitHealthPeriodicLogger( ClusterService clusterService, HealthService testHealthService, - boolean enabled, + boolean started, BiConsumer metricWriter, Consumer logWriter ) { @@ -626,9 +830,13 @@ private HealthPeriodicLogger createAndInitHealthPeriodicLogger( provider ); } - if (enabled) { - clusterSettings.applySettings(Settings.builder().put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true).build()); + if (started) { + testHealthPeriodicLogger.start(); } + // Reset cluster setting + clusterSettings.applySettings(Settings.EMPTY); + // enable + clusterSettings.applySettings(Settings.builder().put(HealthPeriodicLogger.ENABLED_SETTING.getKey(), true).build()); return testHealthPeriodicLogger; } From c0e931af066f982d43fd6e1fd4684172257b2fbe Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 14 Feb 2024 08:55:05 +0000 Subject: [PATCH 21/78] Detach persistent task execution from `ThreadPool` (#105460) Similar to #99392, #97879 etc, no need to have the `NodePersistentTasksExecutor` look up the executor to use each time, nor does it necessarily need to use a named executor from the `ThreadPool`. This commit pulls the lookup earlier in initialization so we can just use a bare `Executor` instead. --- .../geoip/GeoIpDownloaderTaskExecutor.java | 2 +- .../PersistentTaskInitializationFailureIT.java | 6 +----- .../node/selection/HealthNodeTaskExecutor.java | 2 +- .../NodePersistentTasksExecutor.java | 10 +--------- .../persistent/PersistentTasksExecutor.java | 7 ++++--- .../persistent/StartPersistentTaskAction.java | 2 +- .../upgrades/SystemIndexMigrationExecutor.java | 3 +-- .../PersistentTasksNodeServiceTests.java | 18 +++++++----------- .../persistent/TestPersistentTasksPlugin.java | 2 +- .../ccr/action/ShardFollowTasksExecutor.java | 4 ++-- .../xpack/downsample/Downsample.java | 8 +++++++- .../DownsampleShardPersistentTaskExecutor.java | 11 ++++++----- .../xpack/ml/MachineLearning.java | 2 +- .../action/TransportStartDatafeedAction.java | 8 ++++++-- .../AbstractJobPersistentTasksExecutor.java | 4 ++-- ...portStartDataFrameAnalyticsActionTests.java | 2 ++ .../xpack/rollup/job/RollupJobTask.java | 2 +- .../xpack/shutdown/NodeShutdownTasksIT.java | 2 +- .../TransformPersistentTasksExecutor.java | 2 +- 19 files changed, 47 insertions(+), 50 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java index 322eb0666db07..615d1c37bf0cf 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -97,7 +97,7 @@ public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor> getPersistentTasksExecutor( ClusterService clusterService, @@ -132,10 +129,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws static class FailingInitializationPersistentTaskExecutor extends PersistentTasksExecutor { static final String TASK_NAME = "cluster:admin/persistent/test_init_failure"; - static final String EXECUTOR_NAME = "failing_executor"; FailingInitializationPersistentTaskExecutor() { - super(TASK_NAME, EXECUTOR_NAME); + super(TASK_NAME, r -> fail("execution is unexpected")); } @Override diff --git a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java index 20fa43700d0b1..7b95d76bb0e7d 100644 --- a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java @@ -70,7 +70,7 @@ private HealthNodeTaskExecutor( FeatureService featureService, Settings settings ) { - super(TASK_NAME, ThreadPool.Names.MANAGEMENT); + super(TASK_NAME, clusterService.threadPool().executor(ThreadPool.Names.MANAGEMENT)); this.clusterService = clusterService; this.persistentTasksService = persistentTasksService; this.featureService = featureService; diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index bc098a736c15e..a75c4b3352475 100644 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.threadpool.ThreadPool; /** * This component is responsible for execution of persistent tasks. @@ -17,20 +16,13 @@ * It abstracts away the execution of tasks and greatly simplifies testing of PersistentTasksNodeService */ public class NodePersistentTasksExecutor { - - private final ThreadPool threadPool; - - NodePersistentTasksExecutor(ThreadPool threadPool) { - this.threadPool = threadPool; - } - public void executeTask( final Params params, final @Nullable PersistentTaskState state, final AllocatedPersistentTask task, final PersistentTasksExecutor executor ) { - threadPool.executor(executor.getExecutor()).execute(new AbstractRunnable() { + executor.getExecutor().execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { task.markAsFailed(e); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 743210e44b734..0bdd5999731dd 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.Map; +import java.util.concurrent.Executor; import java.util.function.Predicate; /** @@ -25,10 +26,10 @@ */ public abstract class PersistentTasksExecutor { - private final String executor; + private final Executor executor; private final String taskName; - protected PersistentTasksExecutor(String taskName, String executor) { + protected PersistentTasksExecutor(String taskName, Executor executor) { this.taskName = taskName; this.executor = executor; } @@ -117,7 +118,7 @@ protected String getDescription(PersistentTask taskInProgress) { */ protected abstract void nodeOperation(AllocatedPersistentTask task, Params params, @Nullable PersistentTaskState state); - public String getExecutor() { + public Executor getExecutor() { return executor; } } diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 7dbb458354752..299891c64711a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -163,7 +163,7 @@ public TransportAction( threadPool.executor(ThreadPool.Names.GENERIC) ); this.persistentTasksClusterService = persistentTasksClusterService; - NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool); + NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(); clusterService.addListener( new PersistentTasksNodeService( threadPool, diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java index 874a2d6a48022..24d30963d18d8 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java +++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java @@ -23,7 +23,6 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -52,7 +51,7 @@ public SystemIndexMigrationExecutor( MetadataCreateIndexService metadataCreateIndexService, IndexScopedSettings indexScopedSettings ) { - super(SYSTEM_INDEX_UPGRADE_TASK_NAME, ThreadPool.Names.GENERIC); + super(SYSTEM_INDEX_UPGRADE_TASK_NAME, clusterService.threadPool().generic()); this.client = client; this.clusterService = clusterService; this.systemIndices = systemIndices; diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index ee35491a74d00..4bc37ea380bfd 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -94,7 +94,7 @@ public void testStartTask() { PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getExecutor()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); int nonLocalNodesCount = randomInt(10); // need to account for 5 original tasks on each node and their relocations @@ -208,7 +208,7 @@ public void testParamsStatusAndNodeTaskAreDelegated() throws Exception { PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getExecutor()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); TaskId parentId = new TaskId("cluster", 1); AllocatedPersistentTask nodeTask = new TestPersistentTasksPlugin.TestTask( @@ -276,7 +276,7 @@ public void sendCompletionRequest( }; @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getExecutor()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); when(action.getTaskName()).thenReturn("test"); when(action.createTask(anyLong(), anyString(), anyString(), any(), any(), any())).thenReturn( new TestPersistentTasksPlugin.TestTask(1, "persistent", "test", "", new TaskId("cluster", 1), Collections.emptyMap()) @@ -370,7 +370,7 @@ public void sendCompletionRequest( }; @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getExecutor()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); when(action.getTaskName()).thenReturn("test"); when(action.createTask(anyLong(), anyString(), anyString(), any(), any(), any())).thenReturn( new TestPersistentTasksPlugin.TestTask(1, "persistent", "test", "", new TaskId("cluster", 1), Collections.emptyMap()) @@ -478,7 +478,7 @@ public void sendCompletionRequest( @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getExecutor()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); when(action.createTask(anyLong(), anyString(), anyString(), any(), any(), any())).thenThrow( new RuntimeException("Something went wrong") @@ -559,7 +559,7 @@ private ClusterState removeTask(ClusterState state, String taskId) { .build(); } - private class Execution { + private static class Execution { private final PersistentTaskParams params; private final AllocatedPersistentTask task; @@ -573,11 +573,7 @@ private class Execution { } private class MockExecutor extends NodePersistentTasksExecutor { - private List executions = new ArrayList<>(); - - MockExecutor() { - super(null); - } + private final List executions = new ArrayList<>(); @Override public void executeTask( diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index d0ed4f87dbc58..8545ddc067a8d 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -298,7 +298,7 @@ public static class TestPersistentTasksExecutor extends PersistentTasksExecutor< private static volatile boolean nonClusterStateCondition = true; public TestPersistentTasksExecutor(ClusterService clusterService) { - super(NAME, ThreadPool.Names.GENERIC); + super(NAME, clusterService.threadPool().generic()); this.clusterService = clusterService; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index 05945ff3e79a8..942bee6a9d47b 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -104,10 +104,10 @@ public final class ShardFollowTasksExecutor extends PersistentTasksExecutor> getPersistentTasksExecutor( SettingsModule settingsModule, IndexNameExpressionResolver expressionResolver ) { - return List.of(new DownsampleShardPersistentTaskExecutor(client, DownsampleShardTask.TASK_NAME, DOWNSAMPLE_TASK_THREAD_POOL_NAME)); + return List.of( + new DownsampleShardPersistentTaskExecutor( + client, + DownsampleShardTask.TASK_NAME, + threadPool.executor(DOWNSAMPLE_TASK_THREAD_POOL_NAME) + ) + ); } @Override diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java index fbef15d4c24c7..6f110ace53fc9 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.shard.ShardId; @@ -40,7 +41,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.downsample.DownsampleShardIndexerStatus; import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState; @@ -50,13 +50,14 @@ import java.util.Collection; import java.util.Map; import java.util.Objects; +import java.util.concurrent.Executor; public class DownsampleShardPersistentTaskExecutor extends PersistentTasksExecutor { private static final Logger LOGGER = LogManager.getLogger(DownsampleShardPersistentTaskExecutor.class); private final Client client; - public DownsampleShardPersistentTaskExecutor(final Client client, final String taskName, final String executorName) { - super(taskName, executorName); + public DownsampleShardPersistentTaskExecutor(final Client client, final String taskName, final Executor executor) { + super(taskName, executor); this.client = Objects.requireNonNull(client); } @@ -157,9 +158,9 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( } @Override - public String getExecutor() { + public Executor getExecutor() { // The delegate action forks to the a downsample thread: - return ThreadPool.Names.SAME; + return EsExecutors.DIRECT_EXECUTOR_SERVICE; } private void delegate(final AllocatedPersistentTask task, final DownsampleShardTaskParams params, final BytesRef lastDownsampleTsid) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index fb908bd79bab1..5ef7311179e4f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1348,7 +1348,7 @@ public List> getPersistentTasksExecutor( getLicenseState(), machineLearningExtension.get().includeNodeInfo() ), - new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor(datafeedRunner.get(), expressionResolver), + new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor(datafeedRunner.get(), expressionResolver, threadPool), new TransportStartDataFrameAnalyticsAction.TaskExecutor( settings, client, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index f8b2179eadb31..e7d5d956bb1b0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -473,8 +473,12 @@ public static class StartDatafeedPersistentTasksExecutor extends PersistentTasks private final DatafeedRunner datafeedRunner; private final IndexNameExpressionResolver resolver; - public StartDatafeedPersistentTasksExecutor(DatafeedRunner datafeedRunner, IndexNameExpressionResolver resolver) { - super(MlTasks.DATAFEED_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); + public StartDatafeedPersistentTasksExecutor( + DatafeedRunner datafeedRunner, + IndexNameExpressionResolver resolver, + ThreadPool threadPool + ) { + super(MlTasks.DATAFEED_TASK_NAME, threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)); this.datafeedRunner = datafeedRunner; this.resolver = resolver; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java index becb1ac25fc5f..32543b45259c2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java @@ -92,13 +92,13 @@ public static List verifyIndicesPrimaryShardsAreActive( protected AbstractJobPersistentTasksExecutor( String taskName, - String executor, + String executorName, Settings settings, ClusterService clusterService, MlMemoryTracker memoryTracker, IndexNameExpressionResolver expressionResolver ) { - super(taskName, executor); + super(taskName, clusterService.threadPool().executor(executorName)); this.memoryTracker = Objects.requireNonNull(memoryTracker); this.expressionResolver = Objects.requireNonNull(expressionResolver); this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java index 7f7469192b877..64d1414134f38 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.MlMetadata; @@ -134,6 +135,7 @@ private static TaskExecutor createTaskExecutor() { ) ); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class)); return new TaskExecutor( Settings.EMPTY, diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java index bf979f9deabf0..b2e1ed42440a2 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java @@ -60,7 +60,7 @@ public static class RollupJobPersistentTasksExecutor extends PersistentTasksExec private final ThreadPool threadPool; public RollupJobPersistentTasksExecutor(Client client, SchedulerEngine schedulerEngine, ThreadPool threadPool) { - super(RollupField.TASK_NAME, ThreadPool.Names.GENERIC); + super(RollupField.TASK_NAME, threadPool.generic()); this.client = client; this.schedulerEngine = schedulerEngine; this.threadPool = threadPool; diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java index d6bef01672faf..584ee628e81e5 100644 --- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java +++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java @@ -165,7 +165,7 @@ public static final class TaskExecutor extends PersistentTasksExecutor Date: Wed, 14 Feb 2024 09:24:25 +0000 Subject: [PATCH 22/78] Map old release version id directly to their version, don't specify a range (#105335) --- .../src/main/java/org/elasticsearch/ReleaseVersions.java | 7 ++++--- .../test/java/org/elasticsearch/ReleaseVersionsTests.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ReleaseVersions.java b/server/src/main/java/org/elasticsearch/ReleaseVersions.java index 6e7cf6c428cd6..7b5c8d1d42382 100644 --- a/server/src/main/java/org/elasticsearch/ReleaseVersions.java +++ b/server/src/main/java/org/elasticsearch/ReleaseVersions.java @@ -105,14 +105,15 @@ private static IntFunction lookupFunction(NavigableMap versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class); assertThat(versions.apply(17), equalTo("8.1.2-8.2.0")); - assertThat(versions.apply(9), equalTo("0.0.0-8.0.0")); + assertThat(versions.apply(9), equalTo("0.0.0")); assertThat(versions.apply(24), equalTo("8.2.2-snapshot[24]")); } } From ba4d2f5843dbd2fd9a9aca72c0ba4db1802b4664 Mon Sep 17 00:00:00 2001 From: Dmitry Cherniachenko <2sabio@gmail.com> Date: Wed, 14 Feb 2024 10:29:11 +0100 Subject: [PATCH 23/78] Use Arrays.hashCode() for arrays in Objects.hash() calls (#105175) When an array is passed to Objects.hash() it needs to be wrapped with Arrays.hashCode() for calculating the hash of the array content rather than using the array instance "identity hash code" --- .../admin/indices/alias/IndicesAliasesRequest.java | 13 ++++++++++++- .../action/resync/ResyncReplicationRequest.java | 2 +- .../breaker/HierarchyCircuitBreakerService.java | 3 ++- .../xpack/core/ilm/CopySettingsStep.java | 7 ++++--- .../core/ml/action/DeleteExpiredDataAction.java | 5 +++-- .../ml/inference/results/RawInferenceResults.java | 2 +- .../action/apikey/InvalidateApiKeyRequest.java | 2 +- .../security/authc/esnative/UserAndPassword.java | 4 +++- .../xpack/sql/proto/formatter/SimpleFormatter.java | 4 ++-- .../watcher/notification/email/EmailTemplate.java | 12 +++++++++++- .../notification/slack/message/Attachment.java | 8 ++++---- .../notification/slack/message/SlackMessage.java | 2 +- .../slack/message/SlackMessageDefaults.java | 2 +- .../search/WatcherSearchTemplateRequest.java | 2 +- 14 files changed, 47 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index 499698593a14b..a4f5ee9eb672b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -582,7 +582,18 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(type, indices, aliases, filter, routing, indexRouting, searchRouting, writeIndex, isHidden, mustExist); + return Objects.hash( + type, + Arrays.hashCode(indices), + Arrays.hashCode(aliases), + filter, + routing, + indexRouting, + searchRouting, + writeIndex, + isHidden, + mustExist + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java index 4b3d06a5319ea..1dc15a6015a98 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java +++ b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java @@ -80,7 +80,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(trimAboveSeqNo, maxSeenAutoIdTimestampOnPrimary, operations); + return Objects.hash(trimAboveSeqNo, maxSeenAutoIdTimestampOnPrimary, Arrays.hashCode(operations)); } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index a7541dbb50491..a2e30b9e18098 100644 --- a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -601,7 +601,7 @@ static class G1OverLimitStrategy implements OverLimitStrategy { } static long fallbackRegionSize(JvmInfo jvmInfo) { - // mimick JDK calculation based on JDK 14 source: + // mimic JDK calculation based on JDK 14 source: // https://hg.openjdk.java.net/jdk/jdk14/file/6c954123ee8d/src/hotspot/share/gc/g1/heapRegion.cpp#l65 // notice that newer JDKs will have a slight variant only considering max-heap: // https://hg.openjdk.java.net/jdk/jdk/file/e7d0ec2d06e8/src/hotspot/share/gc/g1/heapRegion.cpp#l67 @@ -739,6 +739,7 @@ private TriggerGCResult tryTriggerGC(MemoryUsage memoryUsed) { if (initialCollectionCount != gcCountSupplier.getAsLong()) { break; } + // noinspection ArrayHashCode - prevent array allocation from being optimized away localBlackHole += new byte[allocationSize].hashCode(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java index 2439c1b7c8834..eddeb1a4cb1b2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import java.util.Arrays; import java.util.Locale; import java.util.Objects; import java.util.function.BiFunction; @@ -58,7 +59,7 @@ public String[] getSettingsKeys() { BiFunction getTargetIndexNameSupplier() { return targetIndexNameSupplier; - }; + } @Override public ClusterState performAction(Index index, ClusterState clusterState) { @@ -115,11 +116,11 @@ public boolean equals(Object o) { CopySettingsStep that = (CopySettingsStep) o; return super.equals(o) && Objects.equals(targetIndexNameSupplier, that.targetIndexNameSupplier) - && Objects.equals(settingsKeys, that.settingsKeys); + && Arrays.equals(settingsKeys, that.settingsKeys); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), targetIndexNameSupplier, settingsKeys); + return Objects.hash(super.hashCode(), targetIndexNameSupplier, Arrays.hashCode(settingsKeys)); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java index feb4b16778966..0dc430ee47a94 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import java.io.IOException; +import java.util.Arrays; import java.util.Objects; public class DeleteExpiredDataAction extends ActionType { @@ -130,13 +131,13 @@ public boolean equals(Object o) { Request request = (Request) o; return Objects.equals(requestsPerSecond, request.requestsPerSecond) && Objects.equals(jobId, request.jobId) - && Objects.equals(expandedJobIds, request.expandedJobIds) + && Arrays.equals(expandedJobIds, request.expandedJobIds) && Objects.equals(timeout, request.timeout); } @Override public int hashCode() { - return Objects.hash(requestsPerSecond, timeout, jobId, expandedJobIds); + return Objects.hash(requestsPerSecond, timeout, jobId, Arrays.hashCode(expandedJobIds)); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java index 6f1e3e423b240..6a6a0c40135bd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java @@ -56,7 +56,7 @@ public boolean equals(Object object) { @Override public int hashCode() { - return Objects.hash(Arrays.hashCode(value), featureImportance); + return Objects.hash(Arrays.hashCode(value), Arrays.deepHashCode(featureImportance)); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java index 43908b6cf13fb..bcd8ff0493f72 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java @@ -246,7 +246,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(realmName, userName, ids, name, ownedByAuthenticatedUser); + return Objects.hash(realmName, userName, Arrays.hashCode(ids), name, ownedByAuthenticatedUser); } private static void validateIds(@Nullable String[] idsToValidate) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java index 0de22235c6757..65e2a39ff44b3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java @@ -10,6 +10,8 @@ import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.user.User; +import java.util.Arrays; + /** * Like User, but includes the hashed password * @@ -51,7 +53,7 @@ public boolean equals(Object o) { @Override public int hashCode() { int result = this.user.hashCode(); - result = 31 * result + passwordHash().hashCode(); + result = 31 * result + Arrays.hashCode(passwordHash()); return result; } diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/formatter/SimpleFormatter.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/formatter/SimpleFormatter.java index f79fe17880fdb..004375d976975 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/formatter/SimpleFormatter.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/formatter/SimpleFormatter.java @@ -138,7 +138,7 @@ private String formatWithoutHeader(StringBuilder sb, List> rows) { } } else { // Trim - sb.append(string.substring(0, width[i] - 1)); + sb.append(string, 0, width[i] - 1); sb.append('~'); } } @@ -175,6 +175,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(width, formatOption); + return Objects.hash(Arrays.hashCode(width), formatOption); } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java index db53cfd013651..5fe1dbe6a5012 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailTemplate.java @@ -196,7 +196,17 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(from, replyTo, priority, to, cc, bcc, subject, textBody, htmlBody); + return Objects.hash( + from, + Arrays.hashCode(replyTo), + priority, + Arrays.hashCode(to), + Arrays.hashCode(cc), + Arrays.hashCode(bcc), + subject, + textBody, + htmlBody + ); } @Override diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/Attachment.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/Attachment.java index f217a475eb0a4..d732e3841c596 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/Attachment.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/Attachment.java @@ -105,10 +105,10 @@ public int hashCode() { title, titleLink, text, - fields, + Arrays.hashCode(fields), imageUrl, thumbUrl, - markdownSupportedFields, + Arrays.hashCode(markdownSupportedFields), actions ); } @@ -311,10 +311,10 @@ public int hashCode() { title, titleLink, text, - fields, + Arrays.hashCode(fields), imageUrl, thumbUrl, - markdownSupportedFields, + Arrays.hashCode(markdownSupportedFields), actions ); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java index bb75290c6f0d2..7eb5ca2fccbb6 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java @@ -184,7 +184,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(from, to, text, icon, attachments, dynamicAttachments); + return Objects.hash(from, Arrays.hashCode(to), text, icon, Arrays.hashCode(attachments), dynamicAttachments); } public SlackMessage render( diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageDefaults.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageDefaults.java index 5fbc4a2dd341f..162e9aae944bf 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageDefaults.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageDefaults.java @@ -52,7 +52,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(to, icon, text, attachment); + return Objects.hash(Arrays.hashCode(to), icon, text, attachment); } static class AttachmentDefaults { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java index 0425206f224da..15208f86a5e2b 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java @@ -285,7 +285,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(indices, searchType, indicesOptions, searchSource, template, restTotalHitsAsInt); + return Objects.hash(Arrays.hashCode(indices), searchType, indicesOptions, searchSource, template, restTotalHitsAsInt); } private static final ParseField INDICES_FIELD = new ParseField("indices"); From bb5eacf012f55a0085351dbee775452877def4c9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 14 Feb 2024 09:40:25 +0000 Subject: [PATCH 24/78] Improve `CreateSnapshotStep` failure message (#105480) The wording is a little awkward, and it'd be more helpful to know _which_ snapshot it was that failed. Also we can use `map` rather than `delegateFailure` here. --- .../xpack/core/ilm/CreateSnapshotStep.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStep.java index 2e2798de2c4bd..7d6397b7c96dd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStep.java @@ -20,7 +20,6 @@ import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotNameAlreadyInUseException; -import java.util.Locale; import java.util.Objects; /** @@ -108,7 +107,8 @@ void createSnapshot(IndexMetadata indexMetadata, ActionListener listene request.waitForCompletion(true); request.includeGlobalState(false); request.masterNodeTimeout(TimeValue.MAX_VALUE); - getClient().admin().cluster().createSnapshot(request, listener.delegateFailureAndWrap((l, response) -> { + + getClient().admin().cluster().createSnapshot(request, listener.map(response -> { logger.debug( "create snapshot response for policy [{}] and index [{}] is: {}", policyName, @@ -120,18 +120,20 @@ void createSnapshot(IndexMetadata indexMetadata, ActionListener listene // Check that there are no failed shards, since the request may not entirely // fail, but may still have failures (such as in the case of an aborted snapshot) if (snapInfo.failedShards() == 0) { - l.onResponse(true); + return true; } else { - int failures = snapInfo.failedShards(); - int total = snapInfo.totalShards(); - String message = String.format( - Locale.ROOT, - "failed to create snapshot successfully, %s failures out of %s total shards failed", - failures, - total + logger.warn( + Strings.format( + "failed to create snapshot [%s:%s] for policy [%s] and index [%s]: %s of %s shards failed", + snapshotRepository, + snapshotName, + policyName, + indexName, + snapInfo.failedShards(), + snapInfo.totalShards() + ) ); - logger.warn(message); - l.onResponse(false); + return false; } })); } From 6c3a9d10e33a6a0a1d25ac263ff76df2c1fc310e Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Wed, 14 Feb 2024 12:27:25 +0200 Subject: [PATCH 25/78] Mute 3 SamlAuthenticationIT tests (#105488) testLoginUserWithAuthorizingRealm testLoginUserWithSamlRoleMapping testLoginWithWrongRealmFails Relates #103595 --- .../xpack/security/authc/saml/SamlAuthenticationIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java b/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java index 3f1c532d1adfa..6e6939084cdd3 100644 --- a/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java +++ b/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java @@ -251,6 +251,7 @@ public void setupNativeUser() throws IOException { *
  • Uses that token to verify the user details
    • * */ + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103595") public void testLoginUserWithSamlRoleMapping() throws Exception { final Tuple authTokens = loginViaSaml("shibboleth"); verifyElasticsearchAccessTokenForRoleMapping(authTokens.v1()); @@ -261,6 +262,7 @@ public void testLoginUserWithSamlRoleMapping() throws Exception { verifyElasticsearchAccessTokenInvalidated(accessToken); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103595") public void testLoginUserWithAuthorizingRealm() throws Exception { final Tuple authTokens = loginViaSaml("shibboleth_native"); verifyElasticsearchAccessTokenForAuthorizingRealms(authTokens.v1()); @@ -271,6 +273,7 @@ public void testLoginUserWithAuthorizingRealm() throws Exception { verifyElasticsearchAccessTokenInvalidated(accessToken); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103595") public void testLoginWithWrongRealmFails() throws Exception { final BasicHttpContext context = new BasicHttpContext(); try (CloseableHttpClient client = getHttpClient()) { From de5ec94b29f19cd1bc63c0400d1c6dd9acf48858 Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Wed, 14 Feb 2024 12:41:36 +0200 Subject: [PATCH 26/78] Mute RemoteClusterSecurityBwcRestIT (#105491) Relates #104858 --- .../xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java index fee5129f8c9b8..7c26b8e386cc5 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.remotecluster; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -40,6 +41,7 @@ /** * BWC test which ensures that users and API keys with defined {@code remote_indices} privileges can be used to query legacy remote clusters */ +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/104858") public class RemoteClusterSecurityBwcRestIT extends AbstractRemoteClusterSecurityTestCase { private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version")); From f871cb2364f1620e8046477b905a4e782933a7ff Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Wed, 14 Feb 2024 12:48:32 +0200 Subject: [PATCH 27/78] Mute testClusterResolveDisconnectedAndErrorScenarios (#105492) Relates #105489 --- .../java/org/elasticsearch/indices/cluster/ResolveClusterIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java index c4be9568f8bab..eda16beb33198 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java @@ -522,6 +522,7 @@ public void testClusterResolveWithMatchingAliases() throws IOException { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105489") public void testClusterResolveDisconnectedAndErrorScenarios() throws Exception { Map testClusterInfo = setupThreeClusters(false); String localIndex = (String) testClusterInfo.get("local.index"); From 7c385b001f4999fc11fb3f532e8730fd9a08058e Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 14 Feb 2024 12:09:09 +0100 Subject: [PATCH 28/78] ESQL: Refactor EsqlTranslatorHandler (#105230) * Move the expression translators into their own, dedicated class. * Replace TrivialBinaryComparison and ExpressionTranslators.BinaryComparisons by a single translator handler. --- .../planner/EsqlExpressionTranslators.java | 243 ++++++++++++++++++ .../esql/planner/EsqlTranslatorHandler.java | 217 +--------------- 2 files changed, 244 insertions(+), 216 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java new file mode 100644 index 0000000000000..8ba8efac981af --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.planner; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.InsensitiveEquals; +import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.TypedAttribute; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; +import org.elasticsearch.xpack.ql.planner.ExpressionTranslator; +import org.elasticsearch.xpack.ql.planner.ExpressionTranslators; +import org.elasticsearch.xpack.ql.planner.TranslatorHandler; +import org.elasticsearch.xpack.ql.querydsl.query.MatchAll; +import org.elasticsearch.xpack.ql.querydsl.query.Query; +import org.elasticsearch.xpack.ql.querydsl.query.TermQuery; +import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.Check; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; + +public final class EsqlExpressionTranslators { + + public static final List> QUERY_TRANSLATORS = List.of( + new EqualsIgnoreCaseTranslator(), + new BinaryComparisons(), + new ExpressionTranslators.Ranges(), + new ExpressionTranslators.BinaryLogic(), + new ExpressionTranslators.IsNulls(), + new ExpressionTranslators.IsNotNulls(), + new ExpressionTranslators.Nots(), + new ExpressionTranslators.Likes(), + new ExpressionTranslators.InComparisons(), + new ExpressionTranslators.StringQueries(), + new ExpressionTranslators.Matches(), + new ExpressionTranslators.MultiMatches(), + new Scalars() + ); + + public static Query toQuery(Expression e, TranslatorHandler handler) { + Query translation = null; + for (ExpressionTranslator translator : QUERY_TRANSLATORS) { + translation = translator.translate(e, handler); + if (translation != null) { + return translation; + } + } + + throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e); + } + + public static class EqualsIgnoreCaseTranslator extends ExpressionTranslator { + + @Override + protected Query asQuery(InsensitiveEquals bc, TranslatorHandler handler) { + return doTranslate(bc, handler); + } + + public static Query doTranslate(InsensitiveEquals bc, TranslatorHandler handler) { + checkInsensitiveComparison(bc); + return handler.wrapFunctionQuery(bc, bc.left(), () -> translate(bc)); + } + + public static void checkInsensitiveComparison(InsensitiveEquals bc) { + Check.isTrue( + bc.right().foldable(), + "Line {}:{}: Comparisons against fields are not (currently) supported; offender [{}] in [{}]", + bc.right().sourceLocation().getLineNumber(), + bc.right().sourceLocation().getColumnNumber(), + Expressions.name(bc.right()), + bc.symbol() + ); + } + + static Query translate(InsensitiveEquals bc) { + TypedAttribute attribute = checkIsPushableAttribute(bc.left()); + Source source = bc.source(); + BytesRef value = BytesRefs.toBytesRef(ExpressionTranslators.valueOf(bc.right())); + String name = pushableAttributeName(attribute); + return new TermQuery(source, name, value.utf8ToString(), true); + } + } + + public static class BinaryComparisons extends ExpressionTranslator { + @Override + protected Query asQuery(BinaryComparison bc, TranslatorHandler handler) { + return doTranslate(bc, handler); + } + + public static Query doTranslate(BinaryComparison bc, TranslatorHandler handler) { + ExpressionTranslators.BinaryComparisons.checkBinaryComparison(bc); + Query translated = translateOutOfRangeComparisons(bc); + return translated == null + ? ExpressionTranslators.BinaryComparisons.doTranslate(bc, handler) + : handler.wrapFunctionQuery(bc, bc.left(), () -> translated); + } + + private static Query translateOutOfRangeComparisons(BinaryComparison bc) { + if ((bc.left() instanceof FieldAttribute) == false + || bc.left().dataType().isNumeric() == false + || bc.right().foldable() == false) { + return null; + } + Source source = bc.source(); + Object value = ExpressionTranslators.valueOf(bc.right()); + + // Comparisons with multi-values always return null in ESQL. + if (value instanceof List) { + return new MatchAll(source).negate(source); + } + + DataType valueType = bc.right().dataType(); + DataType attributeDataType = bc.left().dataType(); + if (valueType == UNSIGNED_LONG && value instanceof Long ul) { + value = unsignedLongAsNumber(ul); + } + Number num = (Number) value; + if (isInRange(attributeDataType, valueType, num)) { + return null; + } + + if (Double.isNaN(((Number) value).doubleValue())) { + return new MatchAll(source).negate(source); + } + + boolean matchAllOrNone; + if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { + matchAllOrNone = (num.doubleValue() > 0) == false; + } else if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { + matchAllOrNone = (num.doubleValue() > 0); + } else if (bc instanceof Equals || bc instanceof NullEquals) { + matchAllOrNone = false; + } else if (bc instanceof NotEquals) { + matchAllOrNone = true; + } else { + throw new QlIllegalArgumentException("Unknown binary comparison [{}]", bc); + } + + return matchAllOrNone ? new MatchAll(source) : new MatchAll(source).negate(source); + } + + private static final BigDecimal HALF_FLOAT_MAX = BigDecimal.valueOf(65504); + private static final BigDecimal UNSIGNED_LONG_MAX = BigDecimal.valueOf(2).pow(64).subtract(BigDecimal.ONE); + + private static boolean isInRange(DataType numericFieldDataType, DataType valueDataType, Number value) { + double doubleValue = value.doubleValue(); + if (Double.isNaN(doubleValue) || Double.isInfinite(doubleValue)) { + return false; + } + + BigDecimal decimalValue; + if (value instanceof BigInteger bigIntValue) { + // Unsigned longs may be represented as BigInteger. + decimalValue = new BigDecimal(bigIntValue); + } else { + decimalValue = valueDataType.isRational() ? BigDecimal.valueOf(doubleValue) : BigDecimal.valueOf(value.longValue()); + } + + // Determine min/max for dataType. Use BigDecimals as doubles will have rounding errors for long/ulong. + BigDecimal minValue; + BigDecimal maxValue; + if (numericFieldDataType == DataTypes.BYTE) { + minValue = BigDecimal.valueOf(Byte.MIN_VALUE); + maxValue = BigDecimal.valueOf(Byte.MAX_VALUE); + } else if (numericFieldDataType == DataTypes.SHORT) { + minValue = BigDecimal.valueOf(Short.MIN_VALUE); + maxValue = BigDecimal.valueOf(Short.MAX_VALUE); + } else if (numericFieldDataType == DataTypes.INTEGER) { + minValue = BigDecimal.valueOf(Integer.MIN_VALUE); + maxValue = BigDecimal.valueOf(Integer.MAX_VALUE); + } else if (numericFieldDataType == DataTypes.LONG) { + minValue = BigDecimal.valueOf(Long.MIN_VALUE); + maxValue = BigDecimal.valueOf(Long.MAX_VALUE); + } else if (numericFieldDataType == DataTypes.UNSIGNED_LONG) { + minValue = BigDecimal.ZERO; + maxValue = UNSIGNED_LONG_MAX; + } else if (numericFieldDataType == DataTypes.HALF_FLOAT) { + minValue = HALF_FLOAT_MAX.negate(); + maxValue = HALF_FLOAT_MAX; + } else if (numericFieldDataType == DataTypes.FLOAT) { + minValue = BigDecimal.valueOf(-Float.MAX_VALUE); + maxValue = BigDecimal.valueOf(Float.MAX_VALUE); + } else if (numericFieldDataType == DataTypes.DOUBLE || numericFieldDataType == DataTypes.SCALED_FLOAT) { + // Scaled floats are represented as doubles in ESQL. + minValue = BigDecimal.valueOf(-Double.MAX_VALUE); + maxValue = BigDecimal.valueOf(Double.MAX_VALUE); + } else { + throw new QlIllegalArgumentException("Data type [{}] unsupported for numeric range check", numericFieldDataType); + } + + return minValue.compareTo(decimalValue) <= 0 && maxValue.compareTo(decimalValue) >= 0; + } + } + + public static class Scalars extends ExpressionTranslator { + @Override + protected Query asQuery(ScalarFunction f, TranslatorHandler handler) { + return doTranslate(f, handler); + } + + public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) { + if (f instanceof CIDRMatch cm) { + if (cm.ipField() instanceof FieldAttribute fa && Expressions.foldable(cm.matches())) { + String targetFieldName = handler.nameOf(fa.exactAttribute()); + Set set = new LinkedHashSet<>(Expressions.fold(cm.matches())); + + Query query = new TermsQuery(f.source(), targetFieldName, set); + // CIDR_MATCH applies only to single values. + return handler.wrapFunctionQuery(f, cm.ipField(), () -> query); + } + } + + return ExpressionTranslators.Scalars.doTranslate(f, handler); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java index c610421890fc4..730aa75e03a27 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java @@ -7,83 +7,27 @@ package org.elasticsearch.xpack.esql.planner; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; -import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.InsensitiveEquals; -import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; -import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.MetadataAttribute; -import org.elasticsearch.xpack.ql.expression.TypedAttribute; import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.ql.planner.ExpressionTranslator; -import org.elasticsearch.xpack.ql.planner.ExpressionTranslators; import org.elasticsearch.xpack.ql.planner.QlTranslatorHandler; -import org.elasticsearch.xpack.ql.planner.TranslatorHandler; -import org.elasticsearch.xpack.ql.querydsl.query.MatchAll; import org.elasticsearch.xpack.ql.querydsl.query.Query; -import org.elasticsearch.xpack.ql.querydsl.query.TermQuery; -import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery; -import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; -import org.elasticsearch.xpack.ql.util.Check; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; import java.util.function.Supplier; -import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; - public final class EsqlTranslatorHandler extends QlTranslatorHandler { - public static final List> QUERY_TRANSLATORS = List.of( - new EqualsIgnoreCaseTranslator(), - new TrivialBinaryComparisons(), - new ExpressionTranslators.BinaryComparisons(), - new ExpressionTranslators.Ranges(), - new ExpressionTranslators.BinaryLogic(), - new ExpressionTranslators.IsNulls(), - new ExpressionTranslators.IsNotNulls(), - new ExpressionTranslators.Nots(), - new ExpressionTranslators.Likes(), - new ExpressionTranslators.InComparisons(), - new ExpressionTranslators.StringQueries(), - new ExpressionTranslators.Matches(), - new ExpressionTranslators.MultiMatches(), - new Scalars() - ); - @Override public Query asQuery(Expression e) { - Query translation = null; - for (ExpressionTranslator translator : QUERY_TRANSLATORS) { - translation = translator.translate(e, this); - if (translation != null) { - return translation; - } - } - - throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e); + return EsqlExpressionTranslators.toQuery(e, this); } @Override @@ -112,163 +56,4 @@ public Query wrapFunctionQuery(ScalarFunction sf, Expression field, Supplier { - - @Override - protected Query asQuery(InsensitiveEquals bc, TranslatorHandler handler) { - return doTranslate(bc, handler); - } - - public static Query doTranslate(InsensitiveEquals bc, TranslatorHandler handler) { - checkInsensitiveComparison(bc); - return handler.wrapFunctionQuery(bc, bc.left(), () -> translate(bc)); - } - - public static void checkInsensitiveComparison(InsensitiveEquals bc) { - Check.isTrue( - bc.right().foldable(), - "Line {}:{}: Comparisons against fields are not (currently) supported; offender [{}] in [{}]", - bc.right().sourceLocation().getLineNumber(), - bc.right().sourceLocation().getColumnNumber(), - Expressions.name(bc.right()), - bc.symbol() - ); - } - - static Query translate(InsensitiveEquals bc) { - TypedAttribute attribute = checkIsPushableAttribute(bc.left()); - Source source = bc.source(); - BytesRef value = BytesRefs.toBytesRef(ExpressionTranslators.valueOf(bc.right())); - String name = pushableAttributeName(attribute); - return new TermQuery(source, name, value.utf8ToString(), true); - } - } - - public static class TrivialBinaryComparisons extends ExpressionTranslator { - @Override - protected Query asQuery(BinaryComparison bc, TranslatorHandler handler) { - ExpressionTranslators.BinaryComparisons.checkBinaryComparison(bc); - Query translated = translate(bc); - return translated == null ? null : handler.wrapFunctionQuery(bc, bc.left(), () -> translated); - } - - private static Query translate(BinaryComparison bc) { - if ((bc.left() instanceof FieldAttribute) == false - || bc.left().dataType().isNumeric() == false - || bc.right().foldable() == false) { - return null; - } - Source source = bc.source(); - Object value = ExpressionTranslators.valueOf(bc.right()); - - // Comparisons with multi-values always return null in ESQL. - if (value instanceof List) { - return new MatchAll(source).negate(source); - } - - DataType valueType = bc.right().dataType(); - DataType attributeDataType = bc.left().dataType(); - if (valueType == UNSIGNED_LONG && value instanceof Long ul) { - value = unsignedLongAsNumber(ul); - } - Number num = (Number) value; - if (isInRange(attributeDataType, valueType, num)) { - return null; - } - - if (Double.isNaN(((Number) value).doubleValue())) { - return new MatchAll(source).negate(source); - } - - boolean matchAllOrNone; - if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { - matchAllOrNone = (num.doubleValue() > 0) == false; - } else if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { - matchAllOrNone = (num.doubleValue() > 0); - } else if (bc instanceof Equals || bc instanceof NullEquals) { - matchAllOrNone = false; - } else if (bc instanceof NotEquals) { - matchAllOrNone = true; - } else { - throw new QlIllegalArgumentException("Unknown binary comparison [{}]", bc); - } - - return matchAllOrNone ? new MatchAll(source) : new MatchAll(source).negate(source); - } - - private static final BigDecimal HALF_FLOAT_MAX = BigDecimal.valueOf(65504); - private static final BigDecimal UNSIGNED_LONG_MAX = BigDecimal.valueOf(2).pow(64).subtract(BigDecimal.ONE); - - private static boolean isInRange(DataType numericFieldDataType, DataType valueDataType, Number value) { - double doubleValue = value.doubleValue(); - if (Double.isNaN(doubleValue) || Double.isInfinite(doubleValue)) { - return false; - } - - BigDecimal decimalValue; - if (value instanceof BigInteger bigIntValue) { - // Unsigned longs may be represented as BigInteger. - decimalValue = new BigDecimal(bigIntValue); - } else { - decimalValue = valueDataType.isRational() ? BigDecimal.valueOf(doubleValue) : BigDecimal.valueOf(value.longValue()); - } - - // Determine min/max for dataType. Use BigDecimals as doubles will have rounding errors for long/ulong. - BigDecimal minValue; - BigDecimal maxValue; - if (numericFieldDataType == DataTypes.BYTE) { - minValue = BigDecimal.valueOf(Byte.MIN_VALUE); - maxValue = BigDecimal.valueOf(Byte.MAX_VALUE); - } else if (numericFieldDataType == DataTypes.SHORT) { - minValue = BigDecimal.valueOf(Short.MIN_VALUE); - maxValue = BigDecimal.valueOf(Short.MAX_VALUE); - } else if (numericFieldDataType == DataTypes.INTEGER) { - minValue = BigDecimal.valueOf(Integer.MIN_VALUE); - maxValue = BigDecimal.valueOf(Integer.MAX_VALUE); - } else if (numericFieldDataType == DataTypes.LONG) { - minValue = BigDecimal.valueOf(Long.MIN_VALUE); - maxValue = BigDecimal.valueOf(Long.MAX_VALUE); - } else if (numericFieldDataType == DataTypes.UNSIGNED_LONG) { - minValue = BigDecimal.ZERO; - maxValue = UNSIGNED_LONG_MAX; - } else if (numericFieldDataType == DataTypes.HALF_FLOAT) { - minValue = HALF_FLOAT_MAX.negate(); - maxValue = HALF_FLOAT_MAX; - } else if (numericFieldDataType == DataTypes.FLOAT) { - minValue = BigDecimal.valueOf(-Float.MAX_VALUE); - maxValue = BigDecimal.valueOf(Float.MAX_VALUE); - } else if (numericFieldDataType == DataTypes.DOUBLE || numericFieldDataType == DataTypes.SCALED_FLOAT) { - // Scaled floats are represented as doubles in ESQL. - minValue = BigDecimal.valueOf(-Double.MAX_VALUE); - maxValue = BigDecimal.valueOf(Double.MAX_VALUE); - } else { - throw new QlIllegalArgumentException("Data type [{}] unsupported for numeric range check", numericFieldDataType); - } - - return minValue.compareTo(decimalValue) <= 0 && maxValue.compareTo(decimalValue) >= 0; - } - } - - public static class Scalars extends ExpressionTranslator { - @Override - protected Query asQuery(ScalarFunction f, TranslatorHandler handler) { - return doTranslate(f, handler); - } - - public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) { - if (f instanceof CIDRMatch cm) { - if (cm.ipField() instanceof FieldAttribute fa && Expressions.foldable(cm.matches())) { - String targetFieldName = handler.nameOf(fa.exactAttribute()); - Set set = new LinkedHashSet<>(Expressions.fold(cm.matches())); - - Query query = new TermsQuery(f.source(), targetFieldName, set); - // CIDR_MATCH applies only to single values. - return handler.wrapFunctionQuery(f, cm.ipField(), () -> query); - } - } - - return ExpressionTranslators.Scalars.doTranslate(f, handler); - } - } } From 91e6fbc6d523056099e9e3d7ce5455089e47a32f Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 14 Feb 2024 12:10:59 +0100 Subject: [PATCH 29/78] [Connector API] Add 8.13 docs (#105456) --- .../connector/apis/connector-apis.asciidoc | 6 + .../apis/list-connectors-api.asciidoc | 31 +++++ ...pdate-connector-configuration-api.asciidoc | 124 ++++++++++++++++-- .../update-connector-index-name-api.asciidoc | 85 ++++++++++++ ...update-connector-service-type-api.asciidoc | 87 ++++++++++++ .../apis/update-connector-status-api.asciidoc | 85 ++++++++++++ .../api/connector.update_index_name.json | 2 +- .../api/connector.update_service_type.json | 2 +- .../api/connector.update_status.json | 2 +- 9 files changed, 407 insertions(+), 17 deletions(-) create mode 100644 docs/reference/connector/apis/update-connector-index-name-api.asciidoc create mode 100644 docs/reference/connector/apis/update-connector-service-type-api.asciidoc create mode 100644 docs/reference/connector/apis/update-connector-status-api.asciidoc diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index eabb531551fe5..6652f159053fa 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -29,10 +29,13 @@ Use the following APIs to manage connectors: * <> * <> * <> +* <> * <> * <> * <> * <> +* <> +* <> [discrete] @@ -70,7 +73,10 @@ include::set-connector-sync-job-stats-api.asciidoc[] include::update-connector-configuration-api.asciidoc[] include::update-connector-error-api.asciidoc[] include::update-connector-filtering-api.asciidoc[] +include::update-connector-index-name-api.asciidoc[] include::update-connector-last-sync-api.asciidoc[] include::update-connector-name-description-api.asciidoc[] include::update-connector-pipeline-api.asciidoc[] include::update-connector-scheduling-api.asciidoc[] +include::update-connector-service-type-api.asciidoc[] +include::update-connector-status-api.asciidoc[] diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc index 9b3fc50690243..6c0279fd8b030 100644 --- a/docs/reference/connector/apis/list-connectors-api.asciidoc +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -29,6 +29,15 @@ Returns information about all stored connectors. `from`:: (Optional, integer) The offset from the first result to fetch. +`index_name`:: +(Optional, string) A comma-separated list of data index names associated with connectors, used to filter search results. + +`connector_name`:: +(Optional, string) A comma-separated list of connector names, used to filter search results. + +`service_type`:: +(Optional, string) A comma-separated list of connector service types, used to filter search results. + [[list-connector-api-example]] ==== {api-examples-title} @@ -74,3 +83,25 @@ The following example lists the first two connectors: ---- GET _connector/?from=0&size=2 ---- + +An example to list connectors associated with `search-google-drive` data index name: + +[source,console] +---- +GET _connector/?index_name=search-google-drive +---- + + +An example to list all connectors with `sharepoint_online` service type: + +[source,console] +---- +GET _connector/?service_type=sharepoint_online +---- + +An example to list all connectors with `sharepoint_online` or `google_drive` service type: + +[source,console] +---- +GET _connector/?service_type=sharepoint_online,google_drive +---- diff --git a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc index 57484c14d0f90..22a823f27cd8e 100644 --- a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc @@ -6,7 +6,7 @@ preview::[] -Updates the `configuration` of a connector. +Updates a connector's `configuration`, allowing for complete schema modifications or individual value updates within a registered configuration schema. [[update-connector-configuration-api-request]] @@ -31,8 +31,11 @@ Updates the `configuration` of a connector. [[update-connector-configuration-api-request-body]] ==== {api-request-body-title} +`values`:: +(Optional, object) Configuration values for the connector, represented as a mapping of configuration fields to their respective values within a registered schema. + `configuration`:: -(Required, object) The configuration for the connector. The configuration field is a map where each key represents a specific configuration field name, and the value is a `ConnectorConfiguration` object. +(Optional, object) The configuration for the connector. The configuration field is a map where each key represents a specific configuration field name, and the value is a `ConnectorConfiguration` object. Each `ConnectorConfiguration` object contains the following attributes: @@ -105,41 +108,94 @@ The following example updates the `configuration` for the connector with ID `my- //// [source, console] -------------------------------------------------- -PUT _connector/my-connector +PUT _connector/my-spo-connector { - "index_name": "search-google-drive", - "name": "My Connector", - "service_type": "google_drive" + "index_name": "search-sharepoint-online", + "name": "Sharepoint Online Connector", + "service_type": "sharepoint_online" +} + +PUT _connector/my-spo-connector/_configuration +{ + "configuration": { + "client_id": { + "default_value": null, + "depends_on": [], + "display": "text", + "label": "Client ID", + "options": [], + "order": 3, + "required": true, + "sensitive": false, + "tooltip": null, + "type": "str", + "ui_restrictions": [], + "validations": [], + "value": null + }, + "secret_value": { + "default_value": null, + "depends_on": [], + "display": "text", + "label": "Secret value", + "options": [], + "order": 4, + "required": true, + "sensitive": true, + "tooltip": null, + "type": "str", + "ui_restrictions": [], + "validations": [], + "value": null + } + } } -------------------------------------------------- // TESTSETUP [source,console] -------------------------------------------------- -DELETE _connector/my-connector +DELETE _connector/my-spo-connector -------------------------------------------------- // TEARDOWN //// +This example demonstrates how to register a `sharepoint_online` connector configuration schema. Note: The example does not cover all the necessary configuration fields for operating the Sharepoint Online connector. + [source,console] ---- -PUT _connector/my-connector/_configuration +PUT _connector/my-spo-connector/_configuration { "configuration": { - "service_account_credentials": { + "client_id": { "default_value": null, "depends_on": [], - "display": "textarea", - "label": "Google Drive service account JSON", + "display": "text", + "label": "Client ID", "options": [], - "order": 1, + "order": 3, + "required": true, + "sensitive": false, + "tooltip": null, + "type": "str", + "ui_restrictions": [], + "validations": [], + "value": null + }, + "secret_value": { + "default_value": null, + "depends_on": [], + "display": "text", + "label": "Secret value", + "options": [], + "order": 4, "required": true, "sensitive": true, - "tooltip": "This connectors authenticates as a service account to synchronize content from Google Drive.", + "tooltip": null, "type": "str", "ui_restrictions": [], "validations": [], - "value": "...service account JSON..." + "value": null } } } @@ -151,3 +207,43 @@ PUT _connector/my-connector/_configuration "result": "updated" } ---- + +An example to update configuration values for the `sharepoint_online` connector: + +[source,console] +---- +PUT _connector/my-spo-connector/_configuration +{ + "values": { + "client_id": "my-client-id", + "secret_value": "super-secret-value" + } +} +---- + +[source,console-result] +---- +{ + "result": "updated" +} +---- + + +An example to update single configuration field of the `sharepoint_online` connector. In this case other configuration values won't change: + +[source,console] +---- +PUT _connector/my-spo-connector/_configuration +{ + "values": { + "secret_value": "new-super-secret-value" + } +} +---- + +[source,console-result] +---- +{ + "result": "updated" +} +---- diff --git a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc new file mode 100644 index 0000000000000..be0595a22dcab --- /dev/null +++ b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc @@ -0,0 +1,85 @@ +[[update-connector-index-name-api]] +=== Update connector index name API +++++ +Update connector index name +++++ + +preview::[] + +Updates the `index_name` field of a connector, specifying the index where the data ingested by the connector is stored. + +[[update-connector-index-name-api-request]] +==== {api-request-title} + +`PUT _connector//_index_name` + +[[update-connector-index-name-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_id` parameter should reference an existing connector. + +[[update-connector-index-name-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[role="child_attributes"] +[[update-connector-index-name-api-request-body]] +==== {api-request-body-title} + +`index_name`:: +(Required, string) Index name where the connector ingests data. Each index name can be associated with at most one connector. + + +[[update-connector-index-name-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector `index_name` field was successfully updated. + +`400`:: +The `connector_id` was not provided or the request payload was malformed. + +`404` (Missing resources):: +No connector matching `connector_id` could be found. + +[[update-connector-index-name-api-example]] +==== {api-examples-title} + +The following example updates the `index_name` field for the connector with ID `my-connector`: + +//// +[source, console] +-------------------------------------------------- +PUT _connector/my-connector +{ + "index_name": "search-google-drive", + "name": "My Connector", + "service_type": "google_drive" +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _connector/my-connector +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] +---- +PUT _connector/my-connector/_index_name +{ + "index_name": "data-from-my-google-drive" +} +---- + +[source,console-result] +---- +{ + "result": "updated" +} +---- diff --git a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc new file mode 100644 index 0000000000000..e23db12cb86dc --- /dev/null +++ b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc @@ -0,0 +1,87 @@ +[[update-connector-service-type-api]] +=== Update connector service type API +++++ +Update connector service type +++++ + +preview::[] + +Updates the `service_type` of a connector. + +[[update-connector-service-type-api-request]] +==== {api-request-title} + +`PUT _connector//_service_type` + +[[update-connector-service-type-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_id` parameter should reference an existing connector. +* The `service_type` must be a valid type as defined by the Connector framework. +** When you change a configured connector's `service_type`, you'll also need to reset its configuration to ensure compatibility. + +[[update-connector-service-type-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[role="child_attributes"] +[[update-connector-service-type-api-request-body]] +==== {api-request-body-title} + +`service_type`:: +(Required, string) A connector service type defined in the https://github.com/elastic/connectors/blob/main/connectors/config.py#L94[Connector framework]. + + +[[update-connector-service-type-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector `service_type` field was successfully updated. + +`400`:: +The `connector_id` was not provided or the request payload was malformed. + +`404` (Missing resources):: +No connector matching `connector_id` could be found. + +[[update-connector-service-type-api-example]] +==== {api-examples-title} + +The following example updates the `service_type` of the connector with ID `my-connector`: + +//// +[source, console] +-------------------------------------------------- +PUT _connector/my-connector +{ + "index_name": "search-google-drive", + "name": "My Connector", + "service_type": "google_drive" +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _connector/my-connector +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] +---- +PUT _connector/my-connector/_service_type +{ + "service_type": "sharepoint_online" +} +---- + +[source,console-result] +---- +{ + "result": "updated" +} +---- diff --git a/docs/reference/connector/apis/update-connector-status-api.asciidoc b/docs/reference/connector/apis/update-connector-status-api.asciidoc new file mode 100644 index 0000000000000..29306b03f6897 --- /dev/null +++ b/docs/reference/connector/apis/update-connector-status-api.asciidoc @@ -0,0 +1,85 @@ +[[update-connector-status-api]] +=== Update connector status API +++++ +Update connector status +++++ + +preview::[] + +Updates the `status` of a connector. + +[[update-connector-status-api-request]] +==== {api-request-title} + +`PUT _connector//_status` + +[[update-connector-status-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_id` parameter should reference an existing connector. +* The change of `status` must be a valid status transition according to the https://github.com/elastic/connectors/blob/main/docs/CONNECTOR_PROTOCOL.md[Connector Protocol]. + +[[update-connector-status-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[role="child_attributes"] +[[update-connector-status-api-request-body]] +==== {api-request-body-title} + +`status`:: +(Required, string) A valid connector status string, defined in the Connector Framework. + +[[update-connector-status-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector `status` field was successfully updated. + +`400`:: +The `connector_id` was not provided, the request payload was malformed, or the given status transition is not supported. + +`404` (Missing resources):: +No connector matching `connector_id` could be found. + +[[update-connector-status-api-example]] +==== {api-examples-title} + +The following example updates the `status` of the connector with ID `my-connector`: + +//// +[source, console] +-------------------------------------------------- +PUT _connector/my-connector +{ + "index_name": "search-google-drive", + "name": "My Connector", + "service_type": "needs_configuration" +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _connector/my-connector +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] +---- +PUT _connector/my-connector/_status +{ + "status": "needs_configuration" +} +---- + +[source,console-result] +---- +{ + "result": "updated" +} +---- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_index_name.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_index_name.json index 92efe70a736b9..97d76f60c0292 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_index_name.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_index_name.json @@ -1,7 +1,7 @@ { "connector.update_index_name": { "documentation": { - "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/connector-apis.html", + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-connector-index-name-api.html", "description": "Updates the index name of the connector." }, "stability": "experimental", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_service_type.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_service_type.json index 779fff1750276..279d93c684783 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_service_type.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_service_type.json @@ -1,7 +1,7 @@ { "connector.update_service_type": { "documentation": { - "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/connector-apis.html", + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-connector-service-type-api.html", "description": "Updates the service type of the connector." }, "stability": "experimental", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_status.json index 159dae2e969ab..ea5e506faad89 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_status.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_status.json @@ -1,7 +1,7 @@ { "connector.update_status": { "documentation": { - "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/connector-apis.html", + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-connector-status-api.html", "description": "Updates the status of the connector." }, "stability": "experimental", From 42aaf085ffbf8320b7359d1cbbde44b1d1a9fc24 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 14 Feb 2024 13:44:12 +0100 Subject: [PATCH 30/78] [Connectors API] Fix typo in class docs (#105487) --- .../syncjob/action/ConnectorSyncJobActionRequest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java index b486e58979999..bb83fd78151df 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java @@ -16,8 +16,9 @@ import java.io.IOException; /** - * Abstract base class for action requests targeting the connector syn job index. Implements {@link org.elasticsearch.action.IndicesRequest} - * to ensure index-level privilege support. This class defines the connectors sync job index as the target for all derived action requests. + * Abstract base class for action requests targeting the connector sync job index. + * Implements {@link org.elasticsearch.action.IndicesRequest} to ensure index-level privilege support. + * This class defines the connectors sync job index as the target for all derived action requests. */ public abstract class ConnectorSyncJobActionRequest extends ActionRequest implements IndicesRequest { From a874f47dd8e66b7204cd35e3623f5e34b588f38d Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 14 Feb 2024 08:46:08 -0500 Subject: [PATCH 31/78] Include better output in profiling & toString for automaton based queries (#105468) We have various automaton based queries that build particular automatons based on their usage. However, the input text isn't part of the `toString` output, nor the usage of the current query (wildcard, prefix,etc.). This commit adds a couple of simple queries to wrap some of our logic to make profiling and other output more readable. Here is an example without this change: ``` #(-(winlog.event_data.TargetUserName:AutomatonQuery { org.apache.lucene.util.automaton.Automaton@2d13c057} winlog.event_data.TargetUserName:AutomatonQuery { org.apache.lucene.util.automaton.Automaton@28daf002} winlog.event_data.TargetUserName:AutomatonQuery { org.apache.lucene.util.automaton.Automaton@43c3d7f8} winlog.event_data.TargetUserName:AutomatonQuery { org.apache.lucene.util.automaton.Automaton@2f52905} winlog.event_data.TargetUserName:AutomatonQuery { org.apache.lucene.util.automaton.Automaton@31d75074}) ``` We have 5 case-insensitive automatons, but we don't know which is which in the profiling output. All we know is the originating field. I don't think we can update `AutomatonQuery` directly as sometimes the automaton created mutates the term (prefix for example) and we lose that we are searching for a prefix. --- docs/changelog/105468.yaml | 5 +++ .../lucene/search/AutomatonQueries.java | 7 ++-- .../search/CaseInsensitivePrefixQuery.java | 34 ++++++++++++++++++ .../search/CaseInsensitiveTermQuery.java | 32 +++++++++++++++++ .../search/CaseInsensitiveWildcardQuery.java | 36 +++++++++++++++++++ .../index/mapper/StringFieldType.java | 25 ++++--------- 6 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 docs/changelog/105468.yaml create mode 100644 server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java create mode 100644 server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveTermQuery.java create mode 100644 server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java diff --git a/docs/changelog/105468.yaml b/docs/changelog/105468.yaml new file mode 100644 index 0000000000000..0de36a71862a4 --- /dev/null +++ b/docs/changelog/105468.yaml @@ -0,0 +1,5 @@ +pr: 105468 +summary: Include better output in profiling & `toString` for automaton based queries +area: Search +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java b/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java index 2a9059a046421..d6463fb28f6cf 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/AutomatonQueries.java @@ -44,18 +44,17 @@ public static Automaton caseInsensitivePrefix(String s) { /** Build an automaton query accepting all terms with the specified prefix, ASCII case insensitive. */ public static AutomatonQuery caseInsensitivePrefixQuery(Term prefix) { - return new AutomatonQuery(prefix, caseInsensitivePrefix(prefix.text())); + return new CaseInsensitivePrefixQuery(prefix); } /** Build an automaton accepting all terms ASCII case insensitive. */ public static AutomatonQuery caseInsensitiveTermQuery(Term term) { - BytesRef prefix = term.bytes(); - return new AutomatonQuery(term, toCaseInsensitiveString(prefix)); + return new CaseInsensitiveTermQuery(term); } /** Build an automaton matching a wildcard pattern, ASCII case insensitive. */ public static AutomatonQuery caseInsensitiveWildcardQuery(Term wildcardquery) { - return new AutomatonQuery(wildcardquery, toCaseInsensitiveWildcardAutomaton(wildcardquery)); + return new CaseInsensitiveWildcardQuery(wildcardquery); } /** String equality with support for wildcards */ diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java new file mode 100644 index 0000000000000..e83edaf1d9e22 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitivePrefixQuery.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.lucene.search; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.search.MultiTermQuery; + +import static org.elasticsearch.common.lucene.search.AutomatonQueries.caseInsensitivePrefix; + +public class CaseInsensitivePrefixQuery extends AutomatonQuery { + public CaseInsensitivePrefixQuery(Term term) { + super(term, caseInsensitivePrefix(term.text())); + } + + public CaseInsensitivePrefixQuery(Term term, int determinizeWorkLimit, boolean isBinary) { + super(term, caseInsensitivePrefix(term.text()), determinizeWorkLimit, isBinary); + } + + public CaseInsensitivePrefixQuery(Term term, int determinizeWorkLimit, boolean isBinary, MultiTermQuery.RewriteMethod rewriteMethod) { + super(term, caseInsensitivePrefix(term.text()), determinizeWorkLimit, isBinary, rewriteMethod); + } + + @Override + public String toString(String field) { + return this.getClass().getSimpleName() + "{" + field + ":" + term.text() + "}"; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveTermQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveTermQuery.java new file mode 100644 index 0000000000000..639cd365a7fe6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveTermQuery.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.lucene.search; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; + +import static org.elasticsearch.common.lucene.search.AutomatonQueries.toCaseInsensitiveString; + +/** + * A case insensitive term query. + */ +public class CaseInsensitiveTermQuery extends AutomatonQuery { + /** + * Constructs a case insensitive term query. + * @param term the term to search for, created into a case insensitive automaton + */ + public CaseInsensitiveTermQuery(Term term) { + super(term, toCaseInsensitiveString(term.bytes())); + } + + @Override + public String toString(String field) { + return this.getClass().getSimpleName() + "{" + field + ":" + term.text() + "}"; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java new file mode 100644 index 0000000000000..9480ce19e6c87 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/CaseInsensitiveWildcardQuery.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.lucene.search; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; + +import static org.elasticsearch.common.lucene.search.AutomatonQueries.toCaseInsensitiveWildcardAutomaton; + +/** + * A case insensitive wildcard query. + */ +public class CaseInsensitiveWildcardQuery extends AutomatonQuery { + /** + * Constructs a case insensitive wildcard query. + * @param term the term to search for, created into a case insensitive wildcard automaton + */ + public CaseInsensitiveWildcardQuery(Term term) { + super(term, toCaseInsensitiveWildcardAutomaton(term)); + } + + public CaseInsensitiveWildcardQuery(Term term, int determinizeWorkLimit, boolean isBinary, RewriteMethod rewriteMethod) { + super(term, toCaseInsensitiveWildcardAutomaton(term), determinizeWorkLimit, isBinary, rewriteMethod); + } + + @Override + public String toString(String field) { + return this.getClass().getSimpleName() + "{" + field + ":" + term.text() + "}"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java index 6f68c2f67bdcd..778c733c745ac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java @@ -10,7 +10,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.Term; -import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; @@ -23,6 +22,8 @@ import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.lucene.search.CaseInsensitivePrefixQuery; +import org.elasticsearch.common.lucene.search.CaseInsensitiveWildcardQuery; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.SearchExecutionContext; @@ -31,8 +32,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.elasticsearch.common.lucene.search.AutomatonQueries.caseInsensitivePrefix; -import static org.elasticsearch.common.lucene.search.AutomatonQueries.toCaseInsensitiveWildcardAutomaton; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; /** Base class for {@link MappedFieldType} implementations that use the same @@ -102,14 +101,8 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool Term prefix = new Term(name(), indexedValueForSearch(value)); if (caseInsensitive) { return method == null - ? new AutomatonQuery(prefix, caseInsensitivePrefix(prefix.text()), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false) - : new AutomatonQuery( - prefix, - caseInsensitivePrefix(prefix.text()), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - false, - method - ); + ? new CaseInsensitivePrefixQuery(prefix, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false) + : new CaseInsensitivePrefixQuery(prefix, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false, method); } return method == null ? new PrefixQuery(prefix) : new PrefixQuery(prefix, method); } @@ -177,14 +170,8 @@ protected Query wildcardQuery( } if (caseInsensitive) { return method == null - ? new AutomatonQuery(term, toCaseInsensitiveWildcardAutomaton(term)) - : new AutomatonQuery( - term, - toCaseInsensitiveWildcardAutomaton(term), - Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, - false, - method - ); + ? new CaseInsensitiveWildcardQuery(term) + : new CaseInsensitiveWildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, false, method); } return method == null ? new WildcardQuery(term) : new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, method); } From 4f070200849ed425d6e5471fc961734b29cb3e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Slobodan=20Adamovi=C4=87?= Date: Wed, 14 Feb 2024 14:49:36 +0100 Subject: [PATCH 32/78] Support excluding roles for SAML realm (#105445) This PR adds a new `exclude_roles` setting for SAML realm. This setting allows to exclude certain roles from being mapped to users that are authenticated via SAML realm - regardless of the configured role mappings. The `exclude_roles` setting supports only explicit role names. Regular expressions and wildcards are not supported. The exclusion is possible only if the role mapping is handled by the SAML realm. Hence, it is not possible to configure it along with `authorization_realms` setting. Note: It is intentional that this setting is not registered in this PR. The registration will be addressed in a separate PR. --- .../security/authc/jwt/JwtRealmSettings.java | 30 +--- .../authc/saml/SamlRealmSettings.java | 46 +++++++ .../authc/support/SecuritySettingsUtil.java | 92 +++++++++++++ .../UnregisteredSettingsIntegTests.java | 13 +- .../xpack/security/authc/saml/SamlRealm.java | 7 +- .../support/mapper/ExcludingRoleMapper.java | 50 +++++++ .../security/authc/saml/SamlRealmTests.java | 130 ++++++++++++++---- .../mapper/ExcludingRoleMapperTests.java | 91 ++++++++++++ 8 files changed, 403 insertions(+), 56 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecuritySettingsUtil.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapper.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapperTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java index b05c81653963a..d755abd507611 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java @@ -29,6 +29,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.xpack.core.security.authc.support.SecuritySettingsUtil.verifyNonNullNotEmpty; + /** * Settings unique to each JWT realm. */ @@ -491,34 +493,6 @@ public Iterator> settings() { public static final Collection> DELEGATED_AUTHORIZATION_REALMS_SETTINGS = DelegatedAuthorizationSettings .getSettings(TYPE); - private static void verifyNonNullNotEmpty(final String key, final String value, final List allowedValues) { - assert value != null : "Invalid null value for [" + key + "]."; - if (value.isEmpty()) { - throw new IllegalArgumentException("Invalid empty value for [" + key + "]."); - } - if (allowedValues != null) { - if (allowedValues.contains(value) == false) { - throw new IllegalArgumentException( - "Invalid value [" + value + "] for [" + key + "]. Allowed values are " + allowedValues + "." - ); - } - } - } - - private static void verifyNonNullNotEmpty(final String key, final List values, final List allowedValues) { - assert values != null : "Invalid null list of values for [" + key + "]."; - if (values.isEmpty()) { - if (allowedValues == null) { - throw new IllegalArgumentException("Invalid empty list for [" + key + "]."); - } else { - throw new IllegalArgumentException("Invalid empty list for [" + key + "]. Allowed values are " + allowedValues + "."); - } - } - for (final String value : values) { - verifyNonNullNotEmpty(key, value, allowedValues); - } - } - private static void validateFallbackClaimSetting( Setting.AffixSetting setting, String key, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/saml/SamlRealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/saml/SamlRealmSettings.java index 83d197e78f583..831afcf44a1d0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/saml/SamlRealmSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/saml/SamlRealmSettings.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.security.authc.saml; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.core.security.authc.RealmConfig; @@ -18,9 +19,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.core.security.authc.support.SecuritySettingsUtil.verifyNonNullNotEmpty; + public class SamlRealmSettings { public static final String TYPE = "saml"; @@ -140,6 +145,47 @@ public class SamlRealmSettings { key -> Setting.positiveTimeSetting(key, TimeValue.timeValueMinutes(3), Setting.Property.NodeScope) ); + public static final Setting.AffixSetting> EXCLUDE_ROLES = Setting.affixKeySetting( + RealmSettings.realmSettingPrefix(TYPE), + "exclude_roles", + key -> Setting.stringListSetting(key, new Setting.Validator<>() { + + @Override + public void validate(List excludedRoles) { + excludedRoles.forEach(excludedRole -> verifyNonNullNotEmpty(key, excludedRole)); + } + + @Override + public void validate(List excludedRoles, Map, Object> settings) { + if (false == excludedRoles.isEmpty()) { + final String namespace = EXCLUDE_ROLES.getNamespace(EXCLUDE_ROLES.getConcreteSetting(key)); + final Setting> authorizationRealmsSetting = DelegatedAuthorizationSettings.AUTHZ_REALMS.apply(TYPE) + .getConcreteSettingForNamespace(namespace); + @SuppressWarnings("unchecked") + final List authorizationRealms = (List) settings.get(authorizationRealmsSetting); + if (authorizationRealms != null && false == authorizationRealms.isEmpty()) { + throw new SettingsException( + "Setting [" + + EXCLUDE_ROLES.getConcreteSettingForNamespace(namespace).getKey() + + "] is not permitted when setting [" + + authorizationRealmsSetting.getKey() + + "] is configured." + ); + } + } + } + + @Override + public Iterator> settings() { + final String namespace = EXCLUDE_ROLES.getNamespace(EXCLUDE_ROLES.getConcreteSetting(key)); + final List> settings = List.of( + DelegatedAuthorizationSettings.AUTHZ_REALMS.apply(TYPE).getConcreteSettingForNamespace(namespace) + ); + return settings.iterator(); + } + }, Setting.Property.NodeScope) + ); + public static final String SSL_PREFIX = "ssl."; private SamlRealmSettings() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecuritySettingsUtil.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecuritySettingsUtil.java new file mode 100644 index 0000000000000..fad6e800625f6 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecuritySettingsUtil.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.authc.support; + +import java.util.Collection; +import java.util.List; + +/** + * Utilities for validating security settings. + */ +public final class SecuritySettingsUtil { + + /** + * Validates that a given setting's value is not empty nor null. + * + * @param settingKey The full setting key which is validated. Used for building a proper error messages. + * @param settingValue The value to validate that it's not null nor empty. + */ + public static void verifyNonNullNotEmpty(final String settingKey, final String settingValue) { + verifyNonNullNotEmpty(settingKey, settingValue, null); + } + + /** + * Validates that a given setting's value is not empty nor null and that it is one of the allowed values. + * + * @param settingKey The full setting key which is validated. Used for building a proper error messages. + * @param settingValue The value to validate that it's not null nor empty and that is one of the allowed values. + * @param allowedValues Optional allowed values, against which to validate the given setting value. + * If provided, it will be checked that the setting value is one of these allowed values. + */ + public static void verifyNonNullNotEmpty(final String settingKey, final String settingValue, final Collection allowedValues) { + assert settingValue != null : "Invalid null value for [" + settingKey + "]."; + if (settingValue.isEmpty()) { + throw new IllegalArgumentException("Invalid empty value for [" + settingKey + "]."); + } + if (allowedValues != null) { + if (allowedValues.contains(settingValue) == false) { + throw new IllegalArgumentException( + "Invalid value [" + settingValue + "] for [" + settingKey + "]. Allowed values are " + allowedValues + "." + ); + } + } + } + + /** + * Validates that a given setting's values are not empty nor null. + * + * @param settingKey The full setting key which is validated. Used for building a proper error messages. + * @param settingValues The values to validate that are not null nor empty. + */ + public static void verifyNonNullNotEmpty(final String settingKey, final List settingValues) { + verifyNonNullNotEmpty(settingKey, settingValues, null); + } + + /** + * Validates that a given setting's values are not empty nor null and that are one of the allowed values. + * + * @param settingKey The full setting key which is validated. Used for building a proper error messages. + * @param settingValues The values to validate that are not null nor empty and that are one of the allowed values. + * @param allowedValues The allowed values against which to validate the given setting values. + * If provided, this method will check that the setting values are one of these allowed values. + */ + public static void verifyNonNullNotEmpty( + final String settingKey, + final List settingValues, + final Collection allowedValues + ) { + assert settingValues != null : "Invalid null list of values for [" + settingKey + "]."; + if (settingValues.isEmpty()) { + if (allowedValues == null) { + throw new IllegalArgumentException("Invalid empty list for [" + settingKey + "]."); + } else { + throw new IllegalArgumentException( + "Invalid empty list for [" + settingKey + "]. Allowed values are " + allowedValues + "." + ); + } + } + for (final String settingValue : settingValues) { + verifyNonNullNotEmpty(settingKey, settingValue, allowedValues); + } + } + + private SecuritySettingsUtil() { + throw new IllegalAccessError("not allowed!"); + } + +} diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java index d8cdba122c2fd..c714aa352fd41 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/UnregisteredSettingsIntegTests.java @@ -26,6 +26,17 @@ public void testIncludeReservedRolesSettingNotRegistered() { .putList("xpack.security.reserved_roles.include", "superuser"); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder)); - assertThat(e.getMessage(), containsString("unknown setting")); + assertThat(e.getMessage(), containsString("unknown setting [xpack.security.reserved_roles.include]")); + } + + public void testSamlExcludeRolesSettingNotRegistered() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + + Settings.Builder builder = Settings.builder() + .put(randomBoolean() ? masterNode() : dataOnlyNode()) + .putList("xpack.security.authc.realms.saml.saml1.exclude_roles", "superuser"); + + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder)); + assertThat(e.getMessage(), containsString("unknown setting [xpack.security.authc.realms.saml.saml1.exclude_roles]")); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java index 73c357634d3d4..704875efa18f6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java @@ -52,6 +52,7 @@ import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport; +import org.elasticsearch.xpack.security.authc.support.mapper.ExcludingRoleMapper; import org.opensaml.core.criterion.EntityIdCriterion; import org.opensaml.saml.common.xml.SAMLConstants; import org.opensaml.saml.criterion.EntityRoleCriterion; @@ -261,7 +262,11 @@ public SpConfiguration getServiceProvider() { ) throws Exception { super(config); - this.roleMapper = roleMapper; + if (config.hasSetting(SamlRealmSettings.EXCLUDE_ROLES)) { + this.roleMapper = new ExcludingRoleMapper(roleMapper, config.getSetting(SamlRealmSettings.EXCLUDE_ROLES)); + } else { + this.roleMapper = roleMapper; + } this.authenticator = authenticator; this.logoutHandler = logoutHandler; this.logoutResponseHandler = logoutResponseHandler; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapper.java new file mode 100644 index 0000000000000..70f5213deb676 --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapper.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.authc.support.mapper; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.xpack.core.security.authc.support.CachingRealm; +import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; + +import java.util.Collection; +import java.util.Objects; +import java.util.Set; + +/** + * Implementation of role mapper which wraps a {@link UserRoleMapper} + * and filters out the resolved roles by removing the configured roles to exclude. + */ +public class ExcludingRoleMapper implements UserRoleMapper { + + private final UserRoleMapper delegate; + private final Set rolesToExclude; + + public ExcludingRoleMapper(UserRoleMapper delegate, Collection rolesToExclude) { + this.delegate = Objects.requireNonNull(delegate); + this.rolesToExclude = Set.copyOf(rolesToExclude); + } + + @Override + public void resolveRoles(UserData user, ActionListener> listener) { + delegate.resolveRoles(user, listener.delegateFailureAndWrap((l, r) -> l.onResponse(excludeRoles(r)))); + } + + private Set excludeRoles(Set resolvedRoles) { + if (rolesToExclude.isEmpty()) { + return resolvedRoles; + } else { + return Sets.difference(resolvedRoles, rolesToExclude); + } + } + + @Override + public void refreshRealmOnChange(CachingRealm realm) { + delegate.refreshRealmOnChange(realm); + } +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java index 3ba9b18b24036..2e6ac934a6e8d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java @@ -336,51 +336,72 @@ public void testAbsurdlyLowMinimumRefreshThrowsException() throws Exception { ); } - public void testAuthenticateWithRoleMapping() throws Exception { + private UserRoleMapper mockRoleMapper(Set rolesToReturn, AtomicReference userData) { final UserRoleMapper roleMapper = mock(UserRoleMapper.class); - AtomicReference userData = new AtomicReference<>(); Mockito.doAnswer(invocation -> { assert invocation.getArguments().length == 2; userData.set((UserRoleMapper.UserData) invocation.getArguments()[0]); @SuppressWarnings("unchecked") ActionListener> listener = (ActionListener>) invocation.getArguments()[1]; - listener.onResponse(Collections.singleton("superuser")); + listener.onResponse(rolesToReturn); return null; }).when(roleMapper).resolveRoles(any(UserRoleMapper.UserData.class), anyActionListener()); + return roleMapper; + } + + public void testAuthenticateWithEmptyRoleMapping() throws Exception { + final AtomicReference userData = new AtomicReference<>(); + final UserRoleMapper roleMapper = mockRoleMapper(Set.of(), userData); + final boolean testWithDelimiter = randomBoolean(); + final AuthenticationResult result = performAuthentication( + roleMapper, + randomBoolean(), + randomBoolean(), + randomFrom(Boolean.TRUE, Boolean.FALSE, null), + false, + randomBoolean() ? REALM_NAME : null, + testWithDelimiter ? List.of("STRIKE Team: Delta$shield") : Arrays.asList("avengers", "shield"), + testWithDelimiter ? "$" : null, + randomBoolean() ? List.of("superuser", "kibana_admin") : randomFrom(List.of(), null) + ); + assertThat(result, notNullValue()); + assertThat(result.getStatus(), equalTo(AuthenticationResult.Status.SUCCESS)); + assertThat(result.getValue().roles().length, equalTo(0)); + } + + public void testAuthenticateWithRoleMapping() throws Exception { + final AtomicReference userData = new AtomicReference<>(); + final UserRoleMapper roleMapper = mockRoleMapper(Set.of("superuser", "kibana_admin"), userData); + + final boolean excludeRoles = randomBoolean(); + final List rolesToExclude = excludeRoles ? List.of("superuser") : randomFrom(List.of(), null); final boolean useNameId = randomBoolean(); final boolean principalIsEmailAddress = randomBoolean(); final Boolean populateUserMetadata = randomFrom(Boolean.TRUE, Boolean.FALSE, null); final String authenticatingRealm = randomBoolean() ? REALM_NAME : null; final boolean testWithDelimiter = randomBoolean(); - final AuthenticationResult result; + final AuthenticationResult result = performAuthentication( + roleMapper, + useNameId, + principalIsEmailAddress, + populateUserMetadata, + false, + authenticatingRealm, + testWithDelimiter ? List.of("STRIKE Team: Delta$shield") : Arrays.asList("avengers", "shield"), + testWithDelimiter ? "$" : null, + rolesToExclude + ); - if (testWithDelimiter) { - result = performAuthentication( - roleMapper, - useNameId, - principalIsEmailAddress, - populateUserMetadata, - false, - authenticatingRealm, - List.of("STRIKE Team: Delta$shield"), - "$" - ); - } else { - result = performAuthentication( - roleMapper, - useNameId, - principalIsEmailAddress, - populateUserMetadata, - false, - authenticatingRealm - ); - } assertThat(result, notNullValue()); assertThat(result.getStatus(), equalTo(AuthenticationResult.Status.SUCCESS)); assertThat(result.getValue().principal(), equalTo(useNameId ? "clint.barton" : "cbarton")); assertThat(result.getValue().email(), equalTo("cbarton@shield.gov")); - assertThat(result.getValue().roles(), arrayContainingInAnyOrder("superuser")); + if (excludeRoles) { + assertThat(result.getValue().roles(), arrayContainingInAnyOrder("kibana_admin")); + } else { + assertThat(result.getValue().roles(), arrayContainingInAnyOrder("kibana_admin", "superuser")); + } if (populateUserMetadata == Boolean.FALSE) { // TODO : "saml_nameid" should be null too, but the logout code requires it for now. assertThat(result.getValue().metadata().get("saml_uid"), nullValue()); @@ -473,6 +494,30 @@ private AuthenticationResult performAuthentication( String authenticatingRealm, List groups, String groupsDelimiter + ) throws Exception { + return performAuthentication( + roleMapper, + useNameId, + principalIsEmailAddress, + populateUserMetadata, + useAuthorizingRealm, + authenticatingRealm, + groups, + groupsDelimiter, + null + ); + } + + private AuthenticationResult performAuthentication( + UserRoleMapper roleMapper, + boolean useNameId, + boolean principalIsEmailAddress, + Boolean populateUserMetadata, + boolean useAuthorizingRealm, + String authenticatingRealm, + List groups, + String groupsDelimiter, + List rolesToExclude ) throws Exception { final EntityDescriptor idp = mockIdp(); final SpConfiguration sp = new SpConfiguration("", "https://saml/", null, null, null, Collections.emptyList()); @@ -514,6 +559,9 @@ private AuthenticationResult performAuthentication( populateUserMetadata.booleanValue() ); } + if (rolesToExclude != null) { + settingsBuilder.put(getFullSettingKey(REALM_NAME, SamlRealmSettings.EXCLUDE_ROLES), String.join(",", rolesToExclude)); + } if (useAuthorizingRealm) { settingsBuilder.putList( getFullSettingKey(new RealmConfig.RealmIdentifier("saml", REALM_NAME), DelegatedAuthorizationSettings.AUTHZ_REALMS), @@ -742,6 +790,36 @@ public void testSettingPatternWithoutAttributeThrowsSettingsException() throws E assertThat(settingsException.getMessage(), containsString(REALM_SETTINGS_PREFIX + ".attributes.name")); } + public void testSettingExcludeRolesAndAuthorizationRealmsThrowsException() throws Exception { + final Settings realmSettings = Settings.builder() + .putList(getFullSettingKey(REALM_NAME, SamlRealmSettings.EXCLUDE_ROLES), "superuser", "kibana_admin") + .putList( + getFullSettingKey(new RealmConfig.RealmIdentifier("saml", REALM_NAME), DelegatedAuthorizationSettings.AUTHZ_REALMS), + "ldap" + ) + .build(); + final RealmConfig config = buildConfig(realmSettings); + + final UserRoleMapper roleMapper = mock(UserRoleMapper.class); + final SamlAuthenticator authenticator = mock(SamlAuthenticator.class); + final SamlLogoutRequestHandler logoutHandler = mock(SamlLogoutRequestHandler.class); + final EntityDescriptor idp = mockIdp(); + final SpConfiguration sp = new SpConfiguration("", "https://saml/", null, null, null, Collections.emptyList()); + + var e = expectThrows(IllegalArgumentException.class, () -> buildRealm(config, roleMapper, authenticator, logoutHandler, idp, sp)); + + assertThat( + e.getCause().getMessage(), + containsString( + "Setting [" + + REALM_SETTINGS_PREFIX + + ".exclude_roles] is not permitted when setting [" + + REALM_SETTINGS_PREFIX + + ".authorization_realms] is configured." + ) + ); + } + public void testMissingPrincipalSettingThrowsSettingsException() throws Exception { final Settings realmSettings = Settings.EMPTY; final RealmConfig config = buildConfig(realmSettings); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapperTests.java new file mode 100644 index 0000000000000..0a3df418a8fc2 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExcludingRoleMapperTests.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.authc.support.mapper; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authc.support.CachingRealm; +import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; + +import java.util.Set; + +import static org.elasticsearch.test.ActionListenerUtils.anyActionListener; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class ExcludingRoleMapperTests extends ESTestCase { + + public void testSingleRoleExclusion() throws Exception { + final Set rolesToReturn = Set.of("superuser", "kibana_admin", "monitoring_user"); + final UserRoleMapper delegate = mockUserRoleMapper(rolesToReturn); + PlainActionFuture> listener = new PlainActionFuture<>(); + new ExcludingRoleMapper(delegate, Set.of("superuser")).resolveRoles(mock(UserRoleMapper.UserData.class), listener); + assertThat(listener.get(), containsInAnyOrder("kibana_admin", "monitoring_user")); + } + + public void testNoExclusions() throws Exception { + final Set rolesToReturn = Set.of("superuser", "kibana_admin", "monitoring_user"); + final UserRoleMapper delegate = mockUserRoleMapper(rolesToReturn); + PlainActionFuture> listener = new PlainActionFuture<>(); + final Set rolesToExclude = randomSet( + 0, + 5, + () -> randomValueOtherThanMany(rolesToReturn::contains, () -> randomAlphaOfLengthBetween(3, 8)) + ); + new ExcludingRoleMapper(delegate, rolesToExclude).resolveRoles(mock(UserRoleMapper.UserData.class), listener); + assertThat(listener.get(), containsInAnyOrder("superuser", "kibana_admin", "monitoring_user")); + } + + public void testExcludingAllRoles() throws Exception { + final UserRoleMapper delegate = mockUserRoleMapper(Set.of("superuser", "kibana_admin", "monitoring_user")); + PlainActionFuture> listener = new PlainActionFuture<>(); + new ExcludingRoleMapper(delegate, Set.of("superuser", "kibana_admin", "monitoring_user")).resolveRoles( + mock(UserRoleMapper.UserData.class), + listener + ); + assertThat(listener.get().size(), equalTo(0)); + } + + public void testNothingToExclude() throws Exception { + final UserRoleMapper delegate = mockUserRoleMapper(Set.of()); + PlainActionFuture> listener = new PlainActionFuture<>(); + new ExcludingRoleMapper(delegate, Set.of("superuser", "kibana_admin", "monitoring_user")).resolveRoles( + mock(UserRoleMapper.UserData.class), + listener + ); + assertThat(listener.get().size(), equalTo(0)); + } + + public void testRefreshRealmOnChange() { + final UserRoleMapper delegate = mock(UserRoleMapper.class); + final CachingRealm realm = mock(CachingRealm.class); + new ExcludingRoleMapper(delegate, randomSet(0, 5, () -> randomAlphaOfLengthBetween(3, 6))).refreshRealmOnChange(realm); + + verify(delegate, times(1)).refreshRealmOnChange(same(realm)); + verify(delegate, times(0)).resolveRoles(any(UserRoleMapper.UserData.class), anyActionListener()); + } + + private static UserRoleMapper mockUserRoleMapper(Set rolesToReturn) { + final UserRoleMapper delegate = mock(UserRoleMapper.class); + doAnswer(invocation -> { + assert invocation.getArguments().length == 2; + @SuppressWarnings("unchecked") + ActionListener> listener = (ActionListener>) invocation.getArguments()[1]; + listener.onResponse(rolesToReturn); + return null; + }).when(delegate).resolveRoles(any(UserRoleMapper.UserData.class), anyActionListener()); + return delegate; + } +} From 84def3ad853b0533ea70ae4a498dc6d14054817b Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 14 Feb 2024 08:57:37 -0500 Subject: [PATCH 33/78] Async status response should set is_partial that same way that async response does (#104479) --- .../elasticsearch/xpack/search/MutableSearchResponse.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index a0aabae6e7218..7f3099917e9ec 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -347,8 +347,8 @@ synchronized AsyncStatusResponse toStatusResponse(String asyncExecutionId, long if (finalResponse != null) { return new AsyncStatusResponse( asyncExecutionId, - false, - false, + frozen == false, + isPartial, startTime, expirationTime, startTime + finalResponse.getTook().millis(), @@ -363,7 +363,7 @@ synchronized AsyncStatusResponse toStatusResponse(String asyncExecutionId, long if (failure != null) { return new AsyncStatusResponse( asyncExecutionId, - false, + frozen == false, true, startTime, expirationTime, From bb0e8a4d86463c739af8e64a0c3725ecf941af11 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Wed, 14 Feb 2024 16:29:29 +0100 Subject: [PATCH 34/78] additional test logging (#105508) This change enables the following logging for the test: * refreshed cluster info to ensure allocator is seeing correct data * allocator trace logging to check the balance computation is correct * reconciler debug logging to check if there is anything unexpected during reconciliation --- .../allocation/decider/DiskThresholdDeciderIT.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index 19ba26ac2ab71..7b9f89b60ed94 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; @@ -160,7 +161,11 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(shardSizes.getSmallestShardIds())); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105331") + @TestIssueLogging( + value = "org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceReconciler:DEBUG," + + "org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator:TRACE", + issueUrl = "https://github.com/elastic/elasticsearch/issues/105331" + ) public void testRestoreSnapshotAllocationDoesNotExceedWatermarkWithMultipleShards() throws Exception { internalCluster().startMasterOnlyNode(); internalCluster().startDataOnlyNode(); @@ -303,7 +308,8 @@ private static ShardId removeIndexUUID(ShardId shardId) { private void refreshDiskUsage() { final ClusterInfoService clusterInfoService = internalCluster().getCurrentMasterNodeInstance(ClusterInfoService.class); - ClusterInfoServiceUtils.refresh(((InternalClusterInfoService) clusterInfoService)); + var clusterInfo = ClusterInfoServiceUtils.refresh(((InternalClusterInfoService) clusterInfoService)); + logger.info("Refreshed cluster info: {}", clusterInfo); // if the nodes were all under the low watermark already (but unbalanced) then a change in the disk usage doesn't trigger a reroute // even though it's now possible to achieve better balance, so we have to do an explicit reroute. TODO fix this? if (clusterInfoService.getClusterInfo() From 1fca7257b0e217dc98c1437a9b9eb2ac6f987f41 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 14 Feb 2024 17:28:52 +0100 Subject: [PATCH 35/78] Make CoordinateEncoder an abstract class (#105502) --- .../CartesianShapeCoordinateEncoder.java | 64 --------- .../lucene/spatial/CoordinateEncoder.java | 124 ++++++++++++++++-- .../spatial/GeoShapeCoordinateEncoder.java | 57 -------- .../index/fielddata/Tile2DVisitorTests.java | 2 +- 4 files changed, 115 insertions(+), 132 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/lucene/spatial/CartesianShapeCoordinateEncoder.java delete mode 100644 server/src/main/java/org/elasticsearch/lucene/spatial/GeoShapeCoordinateEncoder.java diff --git a/server/src/main/java/org/elasticsearch/lucene/spatial/CartesianShapeCoordinateEncoder.java b/server/src/main/java/org/elasticsearch/lucene/spatial/CartesianShapeCoordinateEncoder.java deleted file mode 100644 index aa043f8c401be..0000000000000 --- a/server/src/main/java/org/elasticsearch/lucene/spatial/CartesianShapeCoordinateEncoder.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.lucene.spatial; - -import org.apache.lucene.geo.XYEncodingUtils; - -final class CartesianShapeCoordinateEncoder implements CoordinateEncoder { - - private int encode(double value) { - if (value == Double.NEGATIVE_INFINITY) { - return Integer.MIN_VALUE; - } - if (value == Double.POSITIVE_INFINITY) { - return Integer.MAX_VALUE; - } - return XYEncodingUtils.encode((float) value); - } - - private double decode(int value) { - if (value == Integer.MIN_VALUE) { - return Double.NEGATIVE_INFINITY; - } - if (value == Integer.MAX_VALUE) { - return Double.POSITIVE_INFINITY; - } - return XYEncodingUtils.decode(value); - } - - @Override - public int encodeX(double x) { - return encode(x); - } - - @Override - public int encodeY(double y) { - return encode(y); - } - - @Override - public double decodeX(int x) { - return decode(x); - } - - @Override - public double decodeY(int y) { - return decode(y); - } - - @Override - public double normalizeX(double x) { - return x; - } - - @Override - public double normalizeY(double y) { - return y; - } -} diff --git a/server/src/main/java/org/elasticsearch/lucene/spatial/CoordinateEncoder.java b/server/src/main/java/org/elasticsearch/lucene/spatial/CoordinateEncoder.java index e10687246277b..25ecca69da759 100644 --- a/server/src/main/java/org/elasticsearch/lucene/spatial/CoordinateEncoder.java +++ b/server/src/main/java/org/elasticsearch/lucene/spatial/CoordinateEncoder.java @@ -8,30 +8,134 @@ package org.elasticsearch.lucene.spatial; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.geo.XYEncodingUtils; +import org.elasticsearch.common.geo.GeoUtils; + /** - * Interface for classes that help encode double-valued spatial coordinates x/y to + * Abstract API for classes that help encode double-valued spatial coordinates x/y to * their integer-encoded serialized form and decode them back */ -public interface CoordinateEncoder { +public abstract class CoordinateEncoder { - CoordinateEncoder GEO = new GeoShapeCoordinateEncoder(); - CoordinateEncoder CARTESIAN = new CartesianShapeCoordinateEncoder(); + /** Encodes lat/lon values into / from lucene encoded format */ + public static final CoordinateEncoder GEO = new GeoShapeCoordinateEncoder(); + /** Encodes arbitrary x/y values in the float space into / from sortable integers */ + public static final CoordinateEncoder CARTESIAN = new CartesianShapeCoordinateEncoder(); /** encode X value */ - int encodeX(double x); + public abstract int encodeX(double x); /** encode Y value */ - int encodeY(double y); + public abstract int encodeY(double y); /** decode X value */ - double decodeX(int x); + public abstract double decodeX(int x); /** decode Y value */ - double decodeY(int y); + public abstract double decodeY(int y); /** normalize X value */ - double normalizeX(double x); + public abstract double normalizeX(double x); /** normalize Y value */ - double normalizeY(double y); + public abstract double normalizeY(double y); + + private static class GeoShapeCoordinateEncoder extends CoordinateEncoder { + + @Override + public int encodeX(double x) { + if (x == Double.NEGATIVE_INFINITY) { + return Integer.MIN_VALUE; + } + if (x == Double.POSITIVE_INFINITY) { + return Integer.MAX_VALUE; + } + return GeoEncodingUtils.encodeLongitude(x); + } + + @Override + public int encodeY(double y) { + if (y == Double.NEGATIVE_INFINITY) { + return Integer.MIN_VALUE; + } + if (y == Double.POSITIVE_INFINITY) { + return Integer.MAX_VALUE; + } + return GeoEncodingUtils.encodeLatitude(y); + } + + @Override + public double decodeX(int x) { + return GeoEncodingUtils.decodeLongitude(x); + } + + @Override + public double decodeY(int y) { + return GeoEncodingUtils.decodeLatitude(y); + } + + @Override + public double normalizeX(double x) { + return GeoUtils.normalizeLon(x); + } + + @Override + public double normalizeY(double y) { + return GeoUtils.normalizeLat(y); + } + } + + private static class CartesianShapeCoordinateEncoder extends CoordinateEncoder { + + private int encode(double value) { + if (value == Double.NEGATIVE_INFINITY) { + return Integer.MIN_VALUE; + } + if (value == Double.POSITIVE_INFINITY) { + return Integer.MAX_VALUE; + } + return XYEncodingUtils.encode((float) value); + } + + private double decode(int value) { + if (value == Integer.MIN_VALUE) { + return Double.NEGATIVE_INFINITY; + } + if (value == Integer.MAX_VALUE) { + return Double.POSITIVE_INFINITY; + } + return XYEncodingUtils.decode(value); + } + + @Override + public int encodeX(double x) { + return encode(x); + } + + @Override + public int encodeY(double y) { + return encode(y); + } + + @Override + public double decodeX(int x) { + return decode(x); + } + + @Override + public double decodeY(int y) { + return decode(y); + } + + @Override + public double normalizeX(double x) { + return x; + } + + @Override + public double normalizeY(double y) { + return y; + } + } } diff --git a/server/src/main/java/org/elasticsearch/lucene/spatial/GeoShapeCoordinateEncoder.java b/server/src/main/java/org/elasticsearch/lucene/spatial/GeoShapeCoordinateEncoder.java deleted file mode 100644 index 29067d41ac9d3..0000000000000 --- a/server/src/main/java/org/elasticsearch/lucene/spatial/GeoShapeCoordinateEncoder.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.lucene.spatial; - -import org.apache.lucene.geo.GeoEncodingUtils; -import org.elasticsearch.common.geo.GeoUtils; - -final class GeoShapeCoordinateEncoder implements CoordinateEncoder { - - @Override - public int encodeX(double x) { - if (x == Double.NEGATIVE_INFINITY) { - return Integer.MIN_VALUE; - } - if (x == Double.POSITIVE_INFINITY) { - return Integer.MAX_VALUE; - } - return GeoEncodingUtils.encodeLongitude(x); - } - - @Override - public int encodeY(double y) { - if (y == Double.NEGATIVE_INFINITY) { - return Integer.MIN_VALUE; - } - if (y == Double.POSITIVE_INFINITY) { - return Integer.MAX_VALUE; - } - return GeoEncodingUtils.encodeLatitude(y); - } - - @Override - public double decodeX(int x) { - return GeoEncodingUtils.decodeLongitude(x); - } - - @Override - public double decodeY(int y) { - return GeoEncodingUtils.decodeLatitude(y); - } - - @Override - public double normalizeX(double x) { - return GeoUtils.normalizeLon(x); - } - - @Override - public double normalizeY(double y) { - return GeoUtils.normalizeLat(y); - } -} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java index b5a83b2cdd658..f886cf7596e5f 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java @@ -291,7 +291,7 @@ static void assertRelation(GeometryDocValueReader reader, Extent extent, GeoRela assertThat(tile2DVisitor.relation(), in(expectedRelation)); } - private static class TestCoordinateEncoder implements CoordinateEncoder { + private static class TestCoordinateEncoder extends CoordinateEncoder { private static final TestCoordinateEncoder INSTANCE = new TestCoordinateEncoder(); From dbc653e4fdb2b4da189ba2d7ec54ec101539ee35 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 14 Feb 2024 08:46:09 -0800 Subject: [PATCH 36/78] Update docs preview link --- .github/workflows/docs-preview-links.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-preview-links.yml b/.github/workflows/docs-preview-links.yml index 0c3a7fbe9eb3c..30e9c57c0ef69 100644 --- a/.github/workflows/docs-preview-links.yml +++ b/.github/workflows/docs-preview-links.yml @@ -18,8 +18,8 @@ jobs: script: | const pr = context.payload.pull_request; const comment = `Documentation preview: - - ✨ [Changed pages](https://${context.repo.repo}_${pr.number}.docs-preview.app.elstc.co/diff)`; - + - ✨ [Changed pages](https://${context.repo.repo}_bk_${pr.number}.docs-preview.app.elstc.co/diff)`; + github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, From b8d708bb96cea341dd10ac338fe9d07faf1c8373 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 14 Feb 2024 09:47:11 -0800 Subject: [PATCH 37/78] Unwrap transport exceptions in exchange service (#105431) We should unwrap TransportException errors; otherwise, we can return them to the caller instead of the actual underlying cause. This becomes important when the underlying cause is a 4xx error, while TransportException is a 5xx error. I found this when running the heap-attack tests --- .../exchange/ExchangeSourceHandler.java | 7 ++++++- .../esql/action/EsqlActionBreakerIT.java | 21 ++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java index d8f4247cb2dda..859b1fc73c3e1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.operator.exchange; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.SubscribableListener; @@ -14,6 +15,7 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasable; import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.transport.TransportException; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -181,7 +183,10 @@ void fetchPage() { loopControl.exited(); } - void onSinkFailed(Exception e) { + void onSinkFailed(Exception originEx) { + final Exception e = originEx instanceof TransportException + ? (originEx.getCause() instanceof Exception cause ? cause : new ElasticsearchException(originEx.getCause())) + : originEx; failure.getAndUpdate(first -> { if (first == null) { return e; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java index 5e1c3128d4076..ead0173576ff7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; @@ -20,6 +21,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.ArrayList; @@ -76,18 +78,31 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .build(); } - @Override - protected EsqlQueryResponse run(EsqlQueryRequest request) { + private EsqlQueryResponse runWithBreaking(EsqlQueryRequest request) throws CircuitBreakingException { setRequestCircuitBreakerLimit(ByteSizeValue.ofBytes(between(256, 2048))); try { return client().execute(EsqlQueryAction.INSTANCE, request).actionGet(2, TimeUnit.MINUTES); } catch (Exception e) { logger.info("request failed", e); ensureBlocksReleased(); + throw e; } finally { setRequestCircuitBreakerLimit(null); } - return super.run(request); + } + + @Override + protected EsqlQueryResponse run(EsqlQueryRequest request) { + try { + return runWithBreaking(request); + } catch (Exception e) { + try (EsqlQueryResponse resp = super.run(request)) { + assertThat(e, instanceOf(CircuitBreakingException.class)); + assertThat(ExceptionsHelper.status(e), equalTo(RestStatus.TOO_MANY_REQUESTS)); + resp.incRef(); + return resp; + } + } } /** From ce8f5441bf9bf8a08d769d18906f9e36895556f0 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 14 Feb 2024 09:48:09 -0800 Subject: [PATCH 38/78] Reserve bytes before fetching page (#105432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running heap-attack tests with multiple nodes can still lead to OOM errors. This is because the transport response messages are not tracked by the circuit breaker. In heap attack tests, pages can be very large (30MB — I will chunk them later), and for each exchange, we use three concurrent channels, resulting in 100MB of untracked memory. This pull request reserves extra bytes for exchange messages. Although this check doesn't fully prevent OOM errors, it makes them unlikely in such cases. --- .../operator/exchange/ExchangeResponse.java | 8 ++++ .../operator/exchange/ExchangeService.java | 45 +++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java index a4c0c88163523..02ba21b7a7d3d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeResponse.java @@ -65,6 +65,14 @@ public Page takePage() { return page; } + public long ramBytesUsedByPage() { + if (page != null) { + return page.ramBytesUsedByBlocks(); + } else { + return 0; + } + } + /** * Returns true if the {@link RemoteSink} is already completed. In this case, the {@link ExchangeSourceHandler} * can stop polling pages and finish itself. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index 20e297cadbc1a..8065ac6f3086e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; /** * {@link ExchangeService} is responsible for exchanging pages between exchange sinks and sources on the same or different nodes. @@ -237,17 +238,40 @@ public RemoteSink newRemoteSink(Task parentTask, String exchangeId, TransportSer return new TransportRemoteSink(transportService, blockFactory, conn, parentTask, exchangeId, executor); } - record TransportRemoteSink( - TransportService transportService, - BlockFactory blockFactory, - Transport.Connection connection, - Task parentTask, - String exchangeId, - Executor responseExecutor - ) implements RemoteSink { + static final class TransportRemoteSink implements RemoteSink { + final TransportService transportService; + final BlockFactory blockFactory; + final Transport.Connection connection; + final Task parentTask; + final String exchangeId; + final Executor responseExecutor; + + final AtomicLong estimatedPageSizeInBytes = new AtomicLong(0L); + + TransportRemoteSink( + TransportService transportService, + BlockFactory blockFactory, + Transport.Connection connection, + Task parentTask, + String exchangeId, + Executor responseExecutor + ) { + this.transportService = transportService; + this.blockFactory = blockFactory; + this.connection = connection; + this.parentTask = parentTask; + this.exchangeId = exchangeId; + this.responseExecutor = responseExecutor; + } @Override public void fetchPageAsync(boolean allSourcesFinished, ActionListener listener) { + final long reservedBytes = estimatedPageSizeInBytes.get(); + if (reservedBytes > 0) { + // This doesn't fully protect ESQL from OOM, but reduces the likelihood. + blockFactory.breaker().addEstimateBytesAndMaybeBreak(reservedBytes, "fetch page"); + listener = ActionListener.runAfter(listener, () -> blockFactory.breaker().addWithoutBreaking(-reservedBytes)); + } transportService.sendChildRequest( connection, EXCHANGE_ACTION_NAME, @@ -256,7 +280,10 @@ public void fetchPageAsync(boolean allSourcesFinished, ActionListener(listener, in -> { try (BlockStreamInput bsi = new BlockStreamInput(in, blockFactory)) { - return new ExchangeResponse(bsi); + final ExchangeResponse resp = new ExchangeResponse(bsi); + final long responseBytes = resp.ramBytesUsedByPage(); + estimatedPageSizeInBytes.getAndUpdate(curr -> Math.max(responseBytes, curr / 2)); + return resp; } }, responseExecutor) ); From 559edc79bf3dd5e4e1b66fc62153b71bc7bae8d4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 14 Feb 2024 18:05:43 +0000 Subject: [PATCH 39/78] Bump to version 8.14.0 --- .backportrc.json | 4 ++-- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 +++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 1 + build-tools-internal/version.properties | 2 +- docs/reference/migration/index.asciidoc | 2 ++ .../reference/migration/migrate_8_14.asciidoc | 20 +++++++++++++++++++ docs/reference/release-notes.asciidoc | 2 ++ docs/reference/release-notes/8.14.0.asciidoc | 8 ++++++++ .../release-notes/highlights.asciidoc | 17 ++++++++-------- .../main/java/org/elasticsearch/Version.java | 3 ++- 13 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 docs/reference/migration/migrate_8_14.asciidoc create mode 100644 docs/reference/release-notes/8.14.0.asciidoc diff --git a/.backportrc.json b/.backportrc.json index c0d0dbc15a821..cb8aa183f7bf9 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,9 +1,9 @@ { "upstream" : "elastic/elasticsearch", - "targetBranchChoices" : [ "main", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], + "targetBranchChoices" : [ "main", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], "targetPRLabels" : [ "backport" ], "branchLabelMapping" : { - "^v8.13.0$" : "main", + "^v8.14.0$" : "main", "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2" } } \ No newline at end of file diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 2bd91b7fe3739..b5981d5ef40f3 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -48,7 +48,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0"] + BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0", "8.14.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index ed00a0655dbd8..bb2b4748b36ef 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1889,6 +1889,22 @@ steps: env: BWC_VERSION: 8.13.0 + - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.14.0 + - group: packaging-tests-windows steps: - label: "{{matrix.image}} / packaging-tests-windows" diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 86dc3c216d060..141975568d353 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -1162,6 +1162,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.13.0 + - label: 8.14.0 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.0#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.14.0 - label: concurrent-search-tests command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests timeout_in_minutes: 420 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 8ac1a60c9530c..4c6349a86b800 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -115,3 +115,4 @@ BWC_VERSION: - "8.12.1" - "8.12.2" - "8.13.0" + - "8.14.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 079f3565880e4..96c111fd46948 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -2,3 +2,4 @@ BWC_VERSION: - "7.17.19" - "8.12.2" - "8.13.0" + - "8.14.0" diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 56f601fc36197..fa73c475e5242 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,4 +1,4 @@ -elasticsearch = 8.13.0 +elasticsearch = 8.14.0 lucene = 9.9.2 bundled_jdk_vendor = openjdk diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 02fa88f409f27..c524380547839 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -1,5 +1,6 @@ include::migration_intro.asciidoc[] +* <> * <> * <> * <> @@ -15,6 +16,7 @@ include::migration_intro.asciidoc[] * <> * <> +include::migrate_8_14.asciidoc[] include::migrate_8_13.asciidoc[] include::migrate_8_12.asciidoc[] include::migrate_8_11.asciidoc[] diff --git a/docs/reference/migration/migrate_8_14.asciidoc b/docs/reference/migration/migrate_8_14.asciidoc new file mode 100644 index 0000000000000..672c472c14325 --- /dev/null +++ b/docs/reference/migration/migrate_8_14.asciidoc @@ -0,0 +1,20 @@ +[[migrating-8.14]] +== Migrating to 8.14 +++++ +8.14 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.14. + +See also <> and <>. + +coming::[8.14.0] + + +[discrete] +[[breaking-changes-8.14]] +=== Breaking changes + +There are no breaking changes in {es} 8.14. + diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 669402c94e9bb..e548459f31216 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,6 +6,7 @@ This section summarizes the changes in each release. +* <> * <> * <> * <> @@ -60,6 +61,7 @@ This section summarizes the changes in each release. -- +include::release-notes/8.14.0.asciidoc[] include::release-notes/8.13.0.asciidoc[] include::release-notes/8.12.1.asciidoc[] include::release-notes/8.12.0.asciidoc[] diff --git a/docs/reference/release-notes/8.14.0.asciidoc b/docs/reference/release-notes/8.14.0.asciidoc new file mode 100644 index 0000000000000..a203c983927cd --- /dev/null +++ b/docs/reference/release-notes/8.14.0.asciidoc @@ -0,0 +1,8 @@ +[[release-notes-8.14.0]] +== {es} version 8.14.0 + +coming[8.14.0] + +Also see <>. + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 0452eca8fbfc9..a3345c8dc3d74 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -11,7 +11,8 @@ For detailed information about this release, see the <> and // Add previous release to the list Other versions: -{ref-bare}/8.12/release-highlights.html[8.12] +{ref-bare}/8.13/release-highlights.html[8.13] +| {ref-bare}/8.12/release-highlights.html[8.12] | {ref-bare}/8.11/release-highlights.html[8.11] | {ref-bare}/8.10/release-highlights.html[8.10] | {ref-bare}/8.9/release-highlights.html[8.9] @@ -27,15 +28,13 @@ Other versions: endif::[] +// The notable-highlights tag marks entries that +// should be featured in the Stack Installation and Upgrade Guide: // tag::notable-highlights[] - -[discrete] -[[ga_release_of_synonyms_api]] -=== GA Release of Synonyms API -Removes the beta label for the Synonyms API to make it GA. - -{es-pull}103223[#103223] - +// [discrete] +// === Heading +// +// Description. // end::notable-highlights[] diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index aee6f91b33edc..c1e12faab9cf8 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -167,7 +167,8 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_12_1 = new Version(8_12_01_99); public static final Version V_8_12_2 = new Version(8_12_02_99); public static final Version V_8_13_0 = new Version(8_13_00_99); - public static final Version CURRENT = V_8_13_0; + public static final Version V_8_14_0 = new Version(8_14_00_99); + public static final Version CURRENT = V_8_14_0; private static final NavigableMap VERSION_IDS; private static final Map VERSION_STRINGS; From dc0f3ace9b14ccbcf02533f1fa3c3e7ba165b8ce Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 14 Feb 2024 13:23:52 -0500 Subject: [PATCH 40/78] Add 8.13 to branches.json --- branches.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/branches.json b/branches.json index 289928f13daf7..dc72956c13f80 100644 --- a/branches.json +++ b/branches.json @@ -4,6 +4,9 @@ { "branch": "main" }, + { + "branch": "8.13" + }, { "branch": "8.12" }, From f753f6bfcf76423b9c29a6acaa342839b5faddce Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 14 Feb 2024 12:24:47 -0800 Subject: [PATCH 41/78] Fix issue when installing multiple test cluster plugins (#105516) --- .../test/cluster/local/AbstractLocalClusterFactory.java | 8 ++++---- .../xpack/inference/InferenceBaseRestTest.java | 2 +- .../xpack/ml/integration/InferenceBaseRestTest.java | 2 +- .../xpack/security/operator/OperatorPrivilegesIT.java | 2 +- .../SqlTestClusterWithRemote.java | 4 ++-- .../xpack/sql/qa/multi_node/SqlTestCluster.java | 2 +- .../xpack/sql/qa/single_node/SqlTestCluster.java | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index e8a72042f7729..9bd3403060b2a 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -152,10 +152,10 @@ public synchronized void start(Version version) { createConfigDirectory(); copyExtraConfigFiles(); // extra config files might be needed for running cli tools like plugin install copyExtraJarFiles(); - installPlugins(); if (distributionDescriptor.getType() == DistributionType.INTEG_TEST) { installModules(); } + installPlugins(); currentVersion = spec.getVersion(); } else { createConfigDirectory(); @@ -591,7 +591,7 @@ private void configureSecurity() { private void installPlugins() { if (spec.getPlugins().isEmpty() == false) { - Pattern pattern = Pattern.compile("(.+)(?:-\\d\\.\\d\\.\\d-SNAPSHOT\\.zip)?"); + Pattern pattern = Pattern.compile("(.+)(?:-\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?\\.zip)"); LOGGER.info("Installing plugins {} into node '{}", spec.getPlugins(), name); List pluginPaths = Arrays.stream(System.getProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP).split(File.pathSeparator)) @@ -603,8 +603,8 @@ private void installPlugins() { .map( pluginName -> pluginPaths.stream() .map(path -> Pair.of(pattern.matcher(path.getFileName().toString()), path)) - .filter(pair -> pair.left.matches()) - .map(p -> p.right.getParent().resolve(p.left.group(1))) + .filter(pair -> pair.left.matches() && pair.left.group(1).equals(pluginName)) + .map(p -> p.right.getParent().resolve(p.left.group(0))) .findFirst() .orElseThrow(() -> { String taskPath = System.getProperty("tests.task"); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index af6cda3a5efa0..11a5bdf045f21 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -35,7 +35,7 @@ public class InferenceBaseRestTest extends ESRestTestCase { .distribution(DistributionType.DEFAULT) .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") - .plugin("org.elasticsearch.xpack.inference.mock.TestInferenceServicePlugin") + .plugin("inference-service-test") .user("x_pack_rest_user", "x-pack-test-password") .build(); diff --git a/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceBaseRestTest.java b/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceBaseRestTest.java index 51838dba082b9..a99969d5755eb 100644 --- a/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceBaseRestTest.java +++ b/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceBaseRestTest.java @@ -31,7 +31,7 @@ public class InferenceBaseRestTest extends ESRestTestCase { .distribution(DistributionType.DEFAULT) .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") - .plugin("org.elasticsearch.xpack.inference.mock.TestInferenceServicePlugin") + .plugin("inference-service-test") .user("x_pack_rest_user", "x-pack-test-password") .build(); diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java index f1cacae374084..6889c81664173 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java @@ -54,7 +54,7 @@ public class OperatorPrivilegesIT extends ESRestTestCase { .setting("xpack.security.http.ssl.enabled", "false") .setting("xpack.security.operator_privileges.enabled", "true") .setting("path.repo", () -> repoDirectory.getRoot().getPath()) - .plugin("org.elasticsearch.xpack.security.operator.OperatorPrivilegesTestPlugin") + .plugin("operator-privileges-test") .rolesFile(Resource.fromClasspath("roles.yml")) .configFile("service_tokens", Resource.fromClasspath("service_tokens")) .configFile("operator_users.yml", Resource.fromClasspath("operator_users.yml")) diff --git a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_cluster_with_security/SqlTestClusterWithRemote.java b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_cluster_with_security/SqlTestClusterWithRemote.java index a6e5baabd98f3..0608e61488e38 100644 --- a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_cluster_with_security/SqlTestClusterWithRemote.java +++ b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_cluster_with_security/SqlTestClusterWithRemote.java @@ -43,7 +43,7 @@ private static ElasticsearchCluster clusterSettings(String remoteAddress) { .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.autoconfiguration.enabled", "false") .user(USER_NAME, PASSWORD) - .plugin(":x-pack:qa:freeze-plugin") + .plugin("freeze-plugin") .build(); } @@ -58,7 +58,7 @@ private static ElasticsearchCluster remoteClusterSettings() { .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.autoconfiguration.enabled", "false") .user(USER_NAME, PASSWORD) - .plugin(":x-pack:qa:freeze-plugin") + .plugin("freeze-plugin") .build(); } diff --git a/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/SqlTestCluster.java b/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/SqlTestCluster.java index 9859be524ce6a..4f740f22393a7 100644 --- a/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/SqlTestCluster.java +++ b/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/SqlTestCluster.java @@ -20,7 +20,7 @@ public static ElasticsearchCluster getCluster() { .setting("xpack.watcher.enabled", "false") .setting("xpack.security.enabled", "false") .setting("xpack.license.self_generated.type", "trial") - .plugin(":x-pack:qa:freeze-plugin") + .plugin("freeze-plugin") .build(); } } diff --git a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/SqlTestCluster.java b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/SqlTestCluster.java index fd06aa0d7d055..2ce6452bb8d93 100644 --- a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/SqlTestCluster.java +++ b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/SqlTestCluster.java @@ -23,7 +23,7 @@ public static ElasticsearchCluster getCluster(boolean enableFreezing) { .setting("xpack.license.self_generated.type", "trial"); if (enableFreezing) { - settings = settings.plugin(":x-pack:qa:freeze-plugin"); + settings = settings.plugin("freeze-plugin"); } return settings.build(); From e443a7b6baed6ad10d3ba1c978c2228d9280b94e Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 14 Feb 2024 23:23:37 +0200 Subject: [PATCH 42/78] ESQL: Introduce expression validation phase (#105477) Introduce dedicated phase through Validate interface that allows interested expressions to perform post logical optimization verification for things like literals/foldables which might be hidden through references after analysis. Fix #105425 --- docs/changelog/105477.yaml | 6 ++ .../xpack/esql/VerificationException.java | 5 ++ .../xpack/esql/capabilities/Validatable.java | 23 ++++++++ .../xpack/esql/expression/Validations.java | 26 +++++++++ .../function/scalar/math/AutoBucket.java | 17 ++++-- .../esql/optimizer/LogicalPlanOptimizer.java | 7 +-- .../xpack/esql/optimizer/LogicalVerifier.java | 36 ++++-------- .../optimizer/LogicalPlanOptimizerTests.java | 2 +- .../xpack/ql/common/Failures.java | 57 +++++++++++++++++++ 9 files changed, 143 insertions(+), 36 deletions(-) create mode 100644 docs/changelog/105477.yaml create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Validations.java create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/common/Failures.java diff --git a/docs/changelog/105477.yaml b/docs/changelog/105477.yaml new file mode 100644 index 0000000000000..f994d38a3f671 --- /dev/null +++ b/docs/changelog/105477.yaml @@ -0,0 +1,6 @@ +pr: 105477 +summary: "ESQL: Introduce expression validation phase" +area: ES|QL +type: enhancement +issues: + - 105425 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/VerificationException.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/VerificationException.java index 42080115cf0e2..67e13bc954d8c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/VerificationException.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/VerificationException.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql; import org.elasticsearch.xpack.ql.common.Failure; +import org.elasticsearch.xpack.ql.common.Failures; import java.util.Collection; @@ -20,4 +21,8 @@ public VerificationException(Collection sources) { super(Failure.failMessage(sources)); } + public VerificationException(Failures failures) { + super(failures.toString()); + } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java new file mode 100644 index 0000000000000..fc77d8dd03745 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.capabilities; + +import org.elasticsearch.xpack.ql.common.Failures; + +/** + * Interface implemented by expressions that require validation post logical optimization, + * when the plan and references have been not just resolved but also replaced. + */ +public interface Validatable { + + /** + * Validates the implementing expression - discovered failures are reported to the given + * {@link Failures} class. + */ + default void validate(Failures failures) {} +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Validations.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Validations.java new file mode 100644 index 0000000000000..37f262a84d2c4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Validations.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression; + +import org.elasticsearch.xpack.ql.common.Failure; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expression.TypeResolution; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; + +public final class Validations { + + private Validations() {} + + /** + * Validates if the given expression is foldable - if not returns a Failure. + */ + public static Failure isFoldable(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { + TypeResolution resolution = TypeResolutions.isFoldable(e, operationName, paramOrd); + return resolution.unresolved() ? Failure.fail(e, resolution.message()) : null; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AutoBucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AutoBucket.java index c7632e4300597..635bee26370e2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AutoBucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AutoBucket.java @@ -13,12 +13,14 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; +import org.elasticsearch.xpack.ql.common.Failures; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Foldables; import org.elasticsearch.xpack.ql.expression.Literal; @@ -32,11 +34,11 @@ import java.util.function.BiFunction; import java.util.function.Function; +import static org.elasticsearch.xpack.esql.expression.Validations.isFoldable; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.FOURTH; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.THIRD; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isInteger; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isNumeric; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isType; @@ -52,7 +54,7 @@ * in the above case we'll pick month long buckets, yielding 12 buckets. *

    */ -public class AutoBucket extends EsqlScalarFunction { +public class AutoBucket extends EsqlScalarFunction implements Validatable { // TODO maybe we should just cover the whole of representable dates here - like ten years, 100 years, 1000 years, all the way up. // That way you never end up with more than the target number of buckets. private static final Rounding LARGEST_HUMAN_DATE_ROUNDING = Rounding.builder(Rounding.DateTimeUnit.YEAR_OF_CENTURY).build(); @@ -188,10 +190,6 @@ private TypeResolution resolveType(BiFunction failures = verifier.verify(optimized); - if (failures.isEmpty() == false) { + Failures failures = verifier.verify(optimized); + if (failures.hasFailures()) { throw new VerificationException(failures); } return optimized; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java index 8ff495103acfc..bf569ee587dbc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java @@ -7,21 +7,11 @@ package org.elasticsearch.xpack.esql.optimizer; -import org.elasticsearch.xpack.esql.expression.function.scalar.math.AutoBucket; +import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.optimizer.OptimizerRules.LogicalPlanDependencyCheck; -import org.elasticsearch.xpack.ql.common.Failure; -import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.common.Failures; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.Set; - -import static org.elasticsearch.xpack.ql.common.Failure.fail; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.FOURTH; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.THIRD; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable; - public final class LogicalVerifier { private static final LogicalPlanDependencyCheck DEPENDENCY_CHECK = new LogicalPlanDependencyCheck(); @@ -30,25 +20,21 @@ public final class LogicalVerifier { private LogicalVerifier() {} /** Verifies the optimized logical plan. */ - public Collection verify(LogicalPlan plan) { - Set failures = new LinkedHashSet<>(); + public Failures verify(LogicalPlan plan) { + Failures failures = new Failures(); plan.forEachUp(p -> { // dependency check // FIXME: re-enable // DEPENDENCY_CHECK.checkPlan(p, failures); - // post optimization folding check - p.forEachExpression(AutoBucket.class, e -> { - Expression.TypeResolution resolution = isFoldable(e.from(), e.sourceText(), THIRD); - if (resolution.unresolved()) { - failures.add(fail(e, resolution.message())); - } - resolution = isFoldable(e.to(), e.sourceText(), FOURTH); - if (resolution.unresolved()) { - failures.add(fail(e, resolution.message())); - } - }); + if (failures.hasFailures() == false) { + p.forEachExpression(ex -> { + if (ex instanceof Validatable v) { + v.validate(failures); + } + }); + } }); return failures; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index a0fef2aeb2e62..e04344ca86732 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -2942,7 +2942,7 @@ public void testLogicalPlanOptimizerVerificationException() { assertTrue(e.getMessage().startsWith("Found ")); final String header = "Found 1 problem\nline "; assertEquals( - "3:8: third argument of [auto_bucket(salary, 10, emp_no, bucket_end)] must be a constant, received [emp_no]", + "3:32: third argument of [auto_bucket(salary, 10, emp_no, bucket_end)] must be a constant, received [emp_no]", e.getMessage().substring(header.length()) ); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/common/Failures.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/common/Failures.java new file mode 100644 index 0000000000000..3cdf419b239b2 --- /dev/null +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/common/Failures.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ql.common; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Objects; + +/** + * Glorified list for managing {@link Failure}s. + */ +public class Failures { + + private final Collection failures; + + public Failures() { + this.failures = new LinkedHashSet<>(); + } + + public Failures add(Failure failure) { + if (failure != null) { + failures.add(failure); + } + return this; + } + + public boolean hasFailures() { + return failures.size() > 0; + } + + public Collection failures() { + return failures; + } + + @Override + public int hashCode() { + return Objects.hash(failures); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Failures failures1 = (Failures) o; + return Objects.equals(failures, failures1.failures); + } + + @Override + public String toString() { + return failures.isEmpty() ? "[]" : Failure.failMessage(failures); + } +} From 343b1ae1ba74fbf2e75c29adddb2790312dd680b Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 14 Feb 2024 23:56:32 +0200 Subject: [PATCH 43/78] ESQL: Fix bug in grammar that allowed spaces inside id pattern (#105476) Change id pattern into a lexer rule and rely on fragments to avoid having whitespaces inside an id pattern Fix #105441 --- docs/changelog/105476.yaml | 6 + .../esql/src/main/antlr/EsqlBaseLexer.g4 | 29 +- .../esql/src/main/antlr/EsqlBaseLexer.tokens | 2 +- .../esql/src/main/antlr/EsqlBaseParser.g4 | 7 +- .../esql/src/main/antlr/EsqlBaseParser.tokens | 2 +- .../xpack/esql/parser/EsqlBaseLexer.interp | 13 +- .../xpack/esql/parser/EsqlBaseLexer.java | 1269 ++++++++------- .../xpack/esql/parser/EsqlBaseParser.interp | 5 +- .../xpack/esql/parser/EsqlBaseParser.java | 1397 ++++++++--------- .../parser/EsqlBaseParserBaseListener.java | 12 - .../parser/EsqlBaseParserBaseVisitor.java | 7 - .../esql/parser/EsqlBaseParserListener.java | 10 - .../esql/parser/EsqlBaseParserVisitor.java | 6 - .../xpack/esql/parser/ExpressionBuilder.java | 96 +- .../xpack/esql/parser/IdentifierBuilder.java | 7 +- .../esql/parser/StatementParserTests.java | 49 + 16 files changed, 1447 insertions(+), 1470 deletions(-) create mode 100644 docs/changelog/105476.yaml diff --git a/docs/changelog/105476.yaml b/docs/changelog/105476.yaml new file mode 100644 index 0000000000000..6520df78520e7 --- /dev/null +++ b/docs/changelog/105476.yaml @@ -0,0 +1,6 @@ +pr: 105476 +summary: "ESQL: Fix bug in grammar that allowed spaces inside id pattern" +area: ES|QL +type: bug +issues: + - 105441 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index f4f221a636791..ee2d449b21184 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -157,10 +157,14 @@ UNQUOTED_IDENTIFIER | (UNDERSCORE | ASPERAND) UNQUOTED_ID_BODY+ ; -QUOTED_IDENTIFIER +fragment QUOTED_ID : BACKQUOTE BACKQUOTE_BLOCK+ BACKQUOTE ; +QUOTED_IDENTIFIER + : QUOTED_ID + ; + EXPR_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN) ; @@ -220,17 +224,13 @@ fragment UNQUOTED_ID_BODY_WITH_PATTERN : (LETTER | DIGIT | UNDERSCORE | ASTERISK) ; -UNQUOTED_ID_PATTERN +fragment UNQUOTED_ID_PATTERN : (LETTER | ASTERISK) UNQUOTED_ID_BODY_WITH_PATTERN* | (UNDERSCORE | ASPERAND) UNQUOTED_ID_BODY_WITH_PATTERN+ ; -PROJECT_UNQUOTED_IDENTIFIER - : UNQUOTED_ID_PATTERN -> type(UNQUOTED_ID_PATTERN) - ; - -PROJECT_QUOTED_IDENTIFIER - : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) +ID_PATTERN + : (UNQUOTED_ID_PATTERN | QUOTED_ID)+ ; PROJECT_LINE_COMMENT @@ -255,13 +255,8 @@ RENAME_DOT: DOT -> type(DOT); AS : 'as'; -RENAME_QUOTED_IDENTIFIER - : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) - ; - -// use the unquoted pattern to let the parser invalidate fields with * -RENAME_UNQUOTED_IDENTIFIER - : UNQUOTED_ID_PATTERN -> type(UNQUOTED_ID_PATTERN) +RENAME_ID_PATTERN + : ID_PATTERN -> type(ID_PATTERN) ; RENAME_LINE_COMMENT @@ -324,8 +319,8 @@ ENRICH_FIELD_DOT: DOT -> type(DOT); ENRICH_FIELD_WITH : WITH -> type(WITH) ; -ENRICH_FIELD_UNQUOTED_IDENTIFIER - : UNQUOTED_ID_PATTERN -> type(UNQUOTED_ID_PATTERN) +ENRICH_FIELD_ID_PATTERN + : ID_PATTERN -> type(ID_PATTERN) ; ENRICH_FIELD_QUOTED_IDENTIFIER diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 1e3bfb20a2c8b..4bf3584737d1d 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -72,7 +72,7 @@ FROM_UNQUOTED_IDENTIFIER=71 FROM_LINE_COMMENT=72 FROM_MULTILINE_COMMENT=73 FROM_WS=74 -UNQUOTED_ID_PATTERN=75 +ID_PATTERN=75 PROJECT_LINE_COMMENT=76 PROJECT_MULTILINE_COMMENT=77 PROJECT_WS=78 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index bb6db56365149..c3391d71daf0b 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -145,12 +145,7 @@ identifier ; identifierPattern - : idPattern+ - ; - -idPattern - : UNQUOTED_ID_PATTERN - | QUOTED_IDENTIFIER + : ID_PATTERN ; constant diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens index 1e3bfb20a2c8b..4bf3584737d1d 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens @@ -72,7 +72,7 @@ FROM_UNQUOTED_IDENTIFIER=71 FROM_LINE_COMMENT=72 FROM_MULTILINE_COMMENT=73 FROM_WS=74 -UNQUOTED_ID_PATTERN=75 +ID_PATTERN=75 PROJECT_LINE_COMMENT=76 PROJECT_MULTILINE_COMMENT=77 PROJECT_WS=78 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 591b0776fff70..20d06df68b12b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -181,7 +181,7 @@ FROM_UNQUOTED_IDENTIFIER FROM_LINE_COMMENT FROM_MULTILINE_COMMENT FROM_WS -UNQUOTED_ID_PATTERN +ID_PATTERN PROJECT_LINE_COMMENT PROJECT_MULTILINE_COMMENT PROJECT_WS @@ -290,6 +290,7 @@ PERCENT OPENING_BRACKET CLOSING_BRACKET UNQUOTED_IDENTIFIER +QUOTED_ID QUOTED_IDENTIFIER EXPR_LINE_COMMENT EXPR_MULTILINE_COMMENT @@ -311,8 +312,7 @@ PROJECT_DOT PROJECT_COMMA UNQUOTED_ID_BODY_WITH_PATTERN UNQUOTED_ID_PATTERN -PROJECT_UNQUOTED_IDENTIFIER -PROJECT_QUOTED_IDENTIFIER +ID_PATTERN PROJECT_LINE_COMMENT PROJECT_MULTILINE_COMMENT PROJECT_WS @@ -321,8 +321,7 @@ RENAME_ASSIGN RENAME_COMMA RENAME_DOT AS -RENAME_QUOTED_IDENTIFIER -RENAME_UNQUOTED_IDENTIFIER +RENAME_ID_PATTERN RENAME_LINE_COMMENT RENAME_MULTILINE_COMMENT RENAME_WS @@ -342,7 +341,7 @@ ENRICH_FIELD_ASSIGN ENRICH_FIELD_COMMA ENRICH_FIELD_DOT ENRICH_FIELD_WITH -ENRICH_FIELD_UNQUOTED_IDENTIFIER +ENRICH_FIELD_ID_PATTERN ENRICH_FIELD_QUOTED_IDENTIFIER ENRICH_FIELD_LINE_COMMENT ENRICH_FIELD_MULTILINE_COMMENT @@ -385,4 +384,4 @@ SHOW_MODE SETTING_MODE atn: -[4, 0, 104, 1153, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 4, 17, 462, 8, 17, 11, 17, 12, 17, 463, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 5, 18, 472, 8, 18, 10, 18, 12, 18, 475, 9, 18, 1, 18, 3, 18, 478, 8, 18, 1, 18, 3, 18, 481, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 490, 8, 19, 10, 19, 12, 19, 493, 9, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 501, 8, 20, 11, 20, 12, 20, 502, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 3, 31, 544, 8, 31, 1, 31, 4, 31, 547, 8, 31, 11, 31, 12, 31, 548, 1, 32, 1, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 3, 34, 558, 8, 34, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 3, 36, 565, 8, 36, 1, 37, 1, 37, 1, 37, 5, 37, 570, 8, 37, 10, 37, 12, 37, 573, 9, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 5, 37, 581, 8, 37, 10, 37, 12, 37, 584, 9, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 3, 37, 591, 8, 37, 1, 37, 3, 37, 594, 8, 37, 3, 37, 596, 8, 37, 1, 38, 4, 38, 599, 8, 38, 11, 38, 12, 38, 600, 1, 39, 4, 39, 604, 8, 39, 11, 39, 12, 39, 605, 1, 39, 1, 39, 5, 39, 610, 8, 39, 10, 39, 12, 39, 613, 9, 39, 1, 39, 1, 39, 4, 39, 617, 8, 39, 11, 39, 12, 39, 618, 1, 39, 4, 39, 622, 8, 39, 11, 39, 12, 39, 623, 1, 39, 1, 39, 5, 39, 628, 8, 39, 10, 39, 12, 39, 631, 9, 39, 3, 39, 633, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 4, 39, 639, 8, 39, 11, 39, 12, 39, 640, 1, 39, 1, 39, 3, 39, 645, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 5, 76, 773, 8, 76, 10, 76, 12, 76, 776, 9, 76, 1, 76, 1, 76, 3, 76, 780, 8, 76, 1, 76, 4, 76, 783, 8, 76, 11, 76, 12, 76, 784, 3, 76, 787, 8, 76, 1, 77, 1, 77, 4, 77, 791, 8, 77, 11, 77, 12, 77, 792, 1, 77, 1, 77, 1, 78, 1, 78, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 3, 87, 842, 8, 87, 1, 88, 4, 88, 845, 8, 88, 11, 88, 12, 88, 846, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 3, 96, 882, 8, 96, 1, 97, 1, 97, 3, 97, 886, 8, 97, 1, 97, 5, 97, 889, 8, 97, 10, 97, 12, 97, 892, 9, 97, 1, 97, 1, 97, 3, 97, 896, 8, 97, 1, 97, 4, 97, 899, 8, 97, 11, 97, 12, 97, 900, 3, 97, 903, 8, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 118, 4, 118, 990, 8, 118, 11, 118, 12, 118, 991, 1, 118, 1, 118, 3, 118, 996, 8, 118, 1, 118, 4, 118, 999, 8, 118, 11, 118, 12, 118, 1000, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 4, 149, 1138, 8, 149, 11, 149, 12, 149, 1139, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 2, 491, 582, 0, 153, 11, 1, 13, 2, 15, 3, 17, 4, 19, 5, 21, 6, 23, 7, 25, 8, 27, 9, 29, 10, 31, 11, 33, 12, 35, 13, 37, 14, 39, 15, 41, 16, 43, 17, 45, 18, 47, 19, 49, 20, 51, 21, 53, 0, 55, 0, 57, 22, 59, 23, 61, 24, 63, 25, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 0, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 64, 163, 65, 165, 66, 167, 67, 169, 68, 171, 69, 173, 0, 175, 0, 177, 0, 179, 0, 181, 0, 183, 70, 185, 0, 187, 71, 189, 0, 191, 72, 193, 73, 195, 74, 197, 0, 199, 0, 201, 0, 203, 0, 205, 75, 207, 0, 209, 0, 211, 76, 213, 77, 215, 78, 217, 0, 219, 0, 221, 0, 223, 0, 225, 79, 227, 0, 229, 0, 231, 80, 233, 81, 235, 82, 237, 0, 239, 0, 241, 83, 243, 84, 245, 0, 247, 85, 249, 0, 251, 0, 253, 86, 255, 87, 257, 88, 259, 0, 261, 0, 263, 0, 265, 0, 267, 0, 269, 0, 271, 0, 273, 89, 275, 90, 277, 91, 279, 0, 281, 0, 283, 0, 285, 0, 287, 92, 289, 93, 291, 94, 293, 0, 295, 95, 297, 96, 299, 97, 301, 98, 303, 99, 305, 0, 307, 100, 309, 101, 311, 102, 313, 103, 315, 104, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1181, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 1, 53, 1, 0, 0, 0, 1, 55, 1, 0, 0, 0, 1, 57, 1, 0, 0, 0, 1, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 2, 63, 1, 0, 0, 0, 2, 85, 1, 0, 0, 0, 2, 87, 1, 0, 0, 0, 2, 89, 1, 0, 0, 0, 2, 91, 1, 0, 0, 0, 2, 93, 1, 0, 0, 0, 2, 95, 1, 0, 0, 0, 2, 97, 1, 0, 0, 0, 2, 99, 1, 0, 0, 0, 2, 101, 1, 0, 0, 0, 2, 103, 1, 0, 0, 0, 2, 105, 1, 0, 0, 0, 2, 107, 1, 0, 0, 0, 2, 109, 1, 0, 0, 0, 2, 111, 1, 0, 0, 0, 2, 113, 1, 0, 0, 0, 2, 115, 1, 0, 0, 0, 2, 117, 1, 0, 0, 0, 2, 119, 1, 0, 0, 0, 2, 121, 1, 0, 0, 0, 2, 123, 1, 0, 0, 0, 2, 125, 1, 0, 0, 0, 2, 127, 1, 0, 0, 0, 2, 129, 1, 0, 0, 0, 2, 131, 1, 0, 0, 0, 2, 133, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 2, 137, 1, 0, 0, 0, 2, 139, 1, 0, 0, 0, 2, 141, 1, 0, 0, 0, 2, 143, 1, 0, 0, 0, 2, 145, 1, 0, 0, 0, 2, 147, 1, 0, 0, 0, 2, 149, 1, 0, 0, 0, 2, 151, 1, 0, 0, 0, 2, 153, 1, 0, 0, 0, 2, 155, 1, 0, 0, 0, 2, 157, 1, 0, 0, 0, 2, 159, 1, 0, 0, 0, 2, 161, 1, 0, 0, 0, 2, 163, 1, 0, 0, 0, 2, 165, 1, 0, 0, 0, 2, 167, 1, 0, 0, 0, 2, 169, 1, 0, 0, 0, 2, 171, 1, 0, 0, 0, 3, 173, 1, 0, 0, 0, 3, 175, 1, 0, 0, 0, 3, 177, 1, 0, 0, 0, 3, 179, 1, 0, 0, 0, 3, 181, 1, 0, 0, 0, 3, 183, 1, 0, 0, 0, 3, 187, 1, 0, 0, 0, 3, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 4, 197, 1, 0, 0, 0, 4, 199, 1, 0, 0, 0, 4, 201, 1, 0, 0, 0, 4, 205, 1, 0, 0, 0, 4, 207, 1, 0, 0, 0, 4, 209, 1, 0, 0, 0, 4, 211, 1, 0, 0, 0, 4, 213, 1, 0, 0, 0, 4, 215, 1, 0, 0, 0, 5, 217, 1, 0, 0, 0, 5, 219, 1, 0, 0, 0, 5, 221, 1, 0, 0, 0, 5, 223, 1, 0, 0, 0, 5, 225, 1, 0, 0, 0, 5, 227, 1, 0, 0, 0, 5, 229, 1, 0, 0, 0, 5, 231, 1, 0, 0, 0, 5, 233, 1, 0, 0, 0, 5, 235, 1, 0, 0, 0, 6, 237, 1, 0, 0, 0, 6, 239, 1, 0, 0, 0, 6, 241, 1, 0, 0, 0, 6, 243, 1, 0, 0, 0, 6, 247, 1, 0, 0, 0, 6, 249, 1, 0, 0, 0, 6, 251, 1, 0, 0, 0, 6, 253, 1, 0, 0, 0, 6, 255, 1, 0, 0, 0, 6, 257, 1, 0, 0, 0, 7, 259, 1, 0, 0, 0, 7, 261, 1, 0, 0, 0, 7, 263, 1, 0, 0, 0, 7, 265, 1, 0, 0, 0, 7, 267, 1, 0, 0, 0, 7, 269, 1, 0, 0, 0, 7, 271, 1, 0, 0, 0, 7, 273, 1, 0, 0, 0, 7, 275, 1, 0, 0, 0, 7, 277, 1, 0, 0, 0, 8, 279, 1, 0, 0, 0, 8, 281, 1, 0, 0, 0, 8, 283, 1, 0, 0, 0, 8, 285, 1, 0, 0, 0, 8, 287, 1, 0, 0, 0, 8, 289, 1, 0, 0, 0, 8, 291, 1, 0, 0, 0, 9, 293, 1, 0, 0, 0, 9, 295, 1, 0, 0, 0, 9, 297, 1, 0, 0, 0, 9, 299, 1, 0, 0, 0, 9, 301, 1, 0, 0, 0, 9, 303, 1, 0, 0, 0, 10, 305, 1, 0, 0, 0, 10, 307, 1, 0, 0, 0, 10, 309, 1, 0, 0, 0, 10, 311, 1, 0, 0, 0, 10, 313, 1, 0, 0, 0, 10, 315, 1, 0, 0, 0, 11, 317, 1, 0, 0, 0, 13, 327, 1, 0, 0, 0, 15, 334, 1, 0, 0, 0, 17, 343, 1, 0, 0, 0, 19, 350, 1, 0, 0, 0, 21, 360, 1, 0, 0, 0, 23, 367, 1, 0, 0, 0, 25, 374, 1, 0, 0, 0, 27, 388, 1, 0, 0, 0, 29, 395, 1, 0, 0, 0, 31, 403, 1, 0, 0, 0, 33, 415, 1, 0, 0, 0, 35, 424, 1, 0, 0, 0, 37, 430, 1, 0, 0, 0, 39, 437, 1, 0, 0, 0, 41, 444, 1, 0, 0, 0, 43, 452, 1, 0, 0, 0, 45, 461, 1, 0, 0, 0, 47, 467, 1, 0, 0, 0, 49, 484, 1, 0, 0, 0, 51, 500, 1, 0, 0, 0, 53, 506, 1, 0, 0, 0, 55, 511, 1, 0, 0, 0, 57, 516, 1, 0, 0, 0, 59, 520, 1, 0, 0, 0, 61, 524, 1, 0, 0, 0, 63, 528, 1, 0, 0, 0, 65, 532, 1, 0, 0, 0, 67, 534, 1, 0, 0, 0, 69, 536, 1, 0, 0, 0, 71, 539, 1, 0, 0, 0, 73, 541, 1, 0, 0, 0, 75, 550, 1, 0, 0, 0, 77, 552, 1, 0, 0, 0, 79, 557, 1, 0, 0, 0, 81, 559, 1, 0, 0, 0, 83, 564, 1, 0, 0, 0, 85, 595, 1, 0, 0, 0, 87, 598, 1, 0, 0, 0, 89, 644, 1, 0, 0, 0, 91, 646, 1, 0, 0, 0, 93, 649, 1, 0, 0, 0, 95, 653, 1, 0, 0, 0, 97, 657, 1, 0, 0, 0, 99, 659, 1, 0, 0, 0, 101, 661, 1, 0, 0, 0, 103, 666, 1, 0, 0, 0, 105, 668, 1, 0, 0, 0, 107, 674, 1, 0, 0, 0, 109, 680, 1, 0, 0, 0, 111, 685, 1, 0, 0, 0, 113, 687, 1, 0, 0, 0, 115, 690, 1, 0, 0, 0, 117, 693, 1, 0, 0, 0, 119, 698, 1, 0, 0, 0, 121, 702, 1, 0, 0, 0, 123, 707, 1, 0, 0, 0, 125, 713, 1, 0, 0, 0, 127, 716, 1, 0, 0, 0, 129, 718, 1, 0, 0, 0, 131, 724, 1, 0, 0, 0, 133, 726, 1, 0, 0, 0, 135, 731, 1, 0, 0, 0, 137, 734, 1, 0, 0, 0, 139, 737, 1, 0, 0, 0, 141, 740, 1, 0, 0, 0, 143, 742, 1, 0, 0, 0, 145, 745, 1, 0, 0, 0, 147, 747, 1, 0, 0, 0, 149, 750, 1, 0, 0, 0, 151, 752, 1, 0, 0, 0, 153, 754, 1, 0, 0, 0, 155, 756, 1, 0, 0, 0, 157, 758, 1, 0, 0, 0, 159, 760, 1, 0, 0, 0, 161, 765, 1, 0, 0, 0, 163, 786, 1, 0, 0, 0, 165, 788, 1, 0, 0, 0, 167, 796, 1, 0, 0, 0, 169, 800, 1, 0, 0, 0, 171, 804, 1, 0, 0, 0, 173, 808, 1, 0, 0, 0, 175, 813, 1, 0, 0, 0, 177, 817, 1, 0, 0, 0, 179, 821, 1, 0, 0, 0, 181, 825, 1, 0, 0, 0, 183, 829, 1, 0, 0, 0, 185, 841, 1, 0, 0, 0, 187, 844, 1, 0, 0, 0, 189, 848, 1, 0, 0, 0, 191, 852, 1, 0, 0, 0, 193, 856, 1, 0, 0, 0, 195, 860, 1, 0, 0, 0, 197, 864, 1, 0, 0, 0, 199, 869, 1, 0, 0, 0, 201, 873, 1, 0, 0, 0, 203, 881, 1, 0, 0, 0, 205, 902, 1, 0, 0, 0, 207, 904, 1, 0, 0, 0, 209, 908, 1, 0, 0, 0, 211, 912, 1, 0, 0, 0, 213, 916, 1, 0, 0, 0, 215, 920, 1, 0, 0, 0, 217, 924, 1, 0, 0, 0, 219, 929, 1, 0, 0, 0, 221, 933, 1, 0, 0, 0, 223, 937, 1, 0, 0, 0, 225, 941, 1, 0, 0, 0, 227, 944, 1, 0, 0, 0, 229, 948, 1, 0, 0, 0, 231, 952, 1, 0, 0, 0, 233, 956, 1, 0, 0, 0, 235, 960, 1, 0, 0, 0, 237, 964, 1, 0, 0, 0, 239, 969, 1, 0, 0, 0, 241, 974, 1, 0, 0, 0, 243, 979, 1, 0, 0, 0, 245, 986, 1, 0, 0, 0, 247, 995, 1, 0, 0, 0, 249, 1002, 1, 0, 0, 0, 251, 1006, 1, 0, 0, 0, 253, 1010, 1, 0, 0, 0, 255, 1014, 1, 0, 0, 0, 257, 1018, 1, 0, 0, 0, 259, 1022, 1, 0, 0, 0, 261, 1028, 1, 0, 0, 0, 263, 1032, 1, 0, 0, 0, 265, 1036, 1, 0, 0, 0, 267, 1040, 1, 0, 0, 0, 269, 1044, 1, 0, 0, 0, 271, 1048, 1, 0, 0, 0, 273, 1052, 1, 0, 0, 0, 275, 1056, 1, 0, 0, 0, 277, 1060, 1, 0, 0, 0, 279, 1064, 1, 0, 0, 0, 281, 1069, 1, 0, 0, 0, 283, 1073, 1, 0, 0, 0, 285, 1077, 1, 0, 0, 0, 287, 1081, 1, 0, 0, 0, 289, 1085, 1, 0, 0, 0, 291, 1089, 1, 0, 0, 0, 293, 1093, 1, 0, 0, 0, 295, 1098, 1, 0, 0, 0, 297, 1103, 1, 0, 0, 0, 299, 1113, 1, 0, 0, 0, 301, 1117, 1, 0, 0, 0, 303, 1121, 1, 0, 0, 0, 305, 1125, 1, 0, 0, 0, 307, 1130, 1, 0, 0, 0, 309, 1137, 1, 0, 0, 0, 311, 1141, 1, 0, 0, 0, 313, 1145, 1, 0, 0, 0, 315, 1149, 1, 0, 0, 0, 317, 318, 5, 100, 0, 0, 318, 319, 5, 105, 0, 0, 319, 320, 5, 115, 0, 0, 320, 321, 5, 115, 0, 0, 321, 322, 5, 101, 0, 0, 322, 323, 5, 99, 0, 0, 323, 324, 5, 116, 0, 0, 324, 325, 1, 0, 0, 0, 325, 326, 6, 0, 0, 0, 326, 12, 1, 0, 0, 0, 327, 328, 5, 100, 0, 0, 328, 329, 5, 114, 0, 0, 329, 330, 5, 111, 0, 0, 330, 331, 5, 112, 0, 0, 331, 332, 1, 0, 0, 0, 332, 333, 6, 1, 1, 0, 333, 14, 1, 0, 0, 0, 334, 335, 5, 101, 0, 0, 335, 336, 5, 110, 0, 0, 336, 337, 5, 114, 0, 0, 337, 338, 5, 105, 0, 0, 338, 339, 5, 99, 0, 0, 339, 340, 5, 104, 0, 0, 340, 341, 1, 0, 0, 0, 341, 342, 6, 2, 2, 0, 342, 16, 1, 0, 0, 0, 343, 344, 5, 101, 0, 0, 344, 345, 5, 118, 0, 0, 345, 346, 5, 97, 0, 0, 346, 347, 5, 108, 0, 0, 347, 348, 1, 0, 0, 0, 348, 349, 6, 3, 0, 0, 349, 18, 1, 0, 0, 0, 350, 351, 5, 101, 0, 0, 351, 352, 5, 120, 0, 0, 352, 353, 5, 112, 0, 0, 353, 354, 5, 108, 0, 0, 354, 355, 5, 97, 0, 0, 355, 356, 5, 105, 0, 0, 356, 357, 5, 110, 0, 0, 357, 358, 1, 0, 0, 0, 358, 359, 6, 4, 3, 0, 359, 20, 1, 0, 0, 0, 360, 361, 5, 102, 0, 0, 361, 362, 5, 114, 0, 0, 362, 363, 5, 111, 0, 0, 363, 364, 5, 109, 0, 0, 364, 365, 1, 0, 0, 0, 365, 366, 6, 5, 4, 0, 366, 22, 1, 0, 0, 0, 367, 368, 5, 103, 0, 0, 368, 369, 5, 114, 0, 0, 369, 370, 5, 111, 0, 0, 370, 371, 5, 107, 0, 0, 371, 372, 1, 0, 0, 0, 372, 373, 6, 6, 0, 0, 373, 24, 1, 0, 0, 0, 374, 375, 5, 105, 0, 0, 375, 376, 5, 110, 0, 0, 376, 377, 5, 108, 0, 0, 377, 378, 5, 105, 0, 0, 378, 379, 5, 110, 0, 0, 379, 380, 5, 101, 0, 0, 380, 381, 5, 115, 0, 0, 381, 382, 5, 116, 0, 0, 382, 383, 5, 97, 0, 0, 383, 384, 5, 116, 0, 0, 384, 385, 5, 115, 0, 0, 385, 386, 1, 0, 0, 0, 386, 387, 6, 7, 0, 0, 387, 26, 1, 0, 0, 0, 388, 389, 5, 107, 0, 0, 389, 390, 5, 101, 0, 0, 390, 391, 5, 101, 0, 0, 391, 392, 5, 112, 0, 0, 392, 393, 1, 0, 0, 0, 393, 394, 6, 8, 1, 0, 394, 28, 1, 0, 0, 0, 395, 396, 5, 108, 0, 0, 396, 397, 5, 105, 0, 0, 397, 398, 5, 109, 0, 0, 398, 399, 5, 105, 0, 0, 399, 400, 5, 116, 0, 0, 400, 401, 1, 0, 0, 0, 401, 402, 6, 9, 0, 0, 402, 30, 1, 0, 0, 0, 403, 404, 5, 109, 0, 0, 404, 405, 5, 118, 0, 0, 405, 406, 5, 95, 0, 0, 406, 407, 5, 101, 0, 0, 407, 408, 5, 120, 0, 0, 408, 409, 5, 112, 0, 0, 409, 410, 5, 97, 0, 0, 410, 411, 5, 110, 0, 0, 411, 412, 5, 100, 0, 0, 412, 413, 1, 0, 0, 0, 413, 414, 6, 10, 5, 0, 414, 32, 1, 0, 0, 0, 415, 416, 5, 114, 0, 0, 416, 417, 5, 101, 0, 0, 417, 418, 5, 110, 0, 0, 418, 419, 5, 97, 0, 0, 419, 420, 5, 109, 0, 0, 420, 421, 5, 101, 0, 0, 421, 422, 1, 0, 0, 0, 422, 423, 6, 11, 6, 0, 423, 34, 1, 0, 0, 0, 424, 425, 5, 114, 0, 0, 425, 426, 5, 111, 0, 0, 426, 427, 5, 119, 0, 0, 427, 428, 1, 0, 0, 0, 428, 429, 6, 12, 0, 0, 429, 36, 1, 0, 0, 0, 430, 431, 5, 115, 0, 0, 431, 432, 5, 104, 0, 0, 432, 433, 5, 111, 0, 0, 433, 434, 5, 119, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 13, 7, 0, 436, 38, 1, 0, 0, 0, 437, 438, 5, 115, 0, 0, 438, 439, 5, 111, 0, 0, 439, 440, 5, 114, 0, 0, 440, 441, 5, 116, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 14, 0, 0, 443, 40, 1, 0, 0, 0, 444, 445, 5, 115, 0, 0, 445, 446, 5, 116, 0, 0, 446, 447, 5, 97, 0, 0, 447, 448, 5, 116, 0, 0, 448, 449, 5, 115, 0, 0, 449, 450, 1, 0, 0, 0, 450, 451, 6, 15, 0, 0, 451, 42, 1, 0, 0, 0, 452, 453, 5, 119, 0, 0, 453, 454, 5, 104, 0, 0, 454, 455, 5, 101, 0, 0, 455, 456, 5, 114, 0, 0, 456, 457, 5, 101, 0, 0, 457, 458, 1, 0, 0, 0, 458, 459, 6, 16, 0, 0, 459, 44, 1, 0, 0, 0, 460, 462, 8, 0, 0, 0, 461, 460, 1, 0, 0, 0, 462, 463, 1, 0, 0, 0, 463, 461, 1, 0, 0, 0, 463, 464, 1, 0, 0, 0, 464, 465, 1, 0, 0, 0, 465, 466, 6, 17, 0, 0, 466, 46, 1, 0, 0, 0, 467, 468, 5, 47, 0, 0, 468, 469, 5, 47, 0, 0, 469, 473, 1, 0, 0, 0, 470, 472, 8, 1, 0, 0, 471, 470, 1, 0, 0, 0, 472, 475, 1, 0, 0, 0, 473, 471, 1, 0, 0, 0, 473, 474, 1, 0, 0, 0, 474, 477, 1, 0, 0, 0, 475, 473, 1, 0, 0, 0, 476, 478, 5, 13, 0, 0, 477, 476, 1, 0, 0, 0, 477, 478, 1, 0, 0, 0, 478, 480, 1, 0, 0, 0, 479, 481, 5, 10, 0, 0, 480, 479, 1, 0, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 1, 0, 0, 0, 482, 483, 6, 18, 8, 0, 483, 48, 1, 0, 0, 0, 484, 485, 5, 47, 0, 0, 485, 486, 5, 42, 0, 0, 486, 491, 1, 0, 0, 0, 487, 490, 3, 49, 19, 0, 488, 490, 9, 0, 0, 0, 489, 487, 1, 0, 0, 0, 489, 488, 1, 0, 0, 0, 490, 493, 1, 0, 0, 0, 491, 492, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 492, 494, 1, 0, 0, 0, 493, 491, 1, 0, 0, 0, 494, 495, 5, 42, 0, 0, 495, 496, 5, 47, 0, 0, 496, 497, 1, 0, 0, 0, 497, 498, 6, 19, 8, 0, 498, 50, 1, 0, 0, 0, 499, 501, 7, 2, 0, 0, 500, 499, 1, 0, 0, 0, 501, 502, 1, 0, 0, 0, 502, 500, 1, 0, 0, 0, 502, 503, 1, 0, 0, 0, 503, 504, 1, 0, 0, 0, 504, 505, 6, 20, 8, 0, 505, 52, 1, 0, 0, 0, 506, 507, 3, 159, 74, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 21, 9, 0, 509, 510, 6, 21, 10, 0, 510, 54, 1, 0, 0, 0, 511, 512, 3, 63, 26, 0, 512, 513, 1, 0, 0, 0, 513, 514, 6, 22, 11, 0, 514, 515, 6, 22, 12, 0, 515, 56, 1, 0, 0, 0, 516, 517, 3, 51, 20, 0, 517, 518, 1, 0, 0, 0, 518, 519, 6, 23, 8, 0, 519, 58, 1, 0, 0, 0, 520, 521, 3, 47, 18, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 24, 8, 0, 523, 60, 1, 0, 0, 0, 524, 525, 3, 49, 19, 0, 525, 526, 1, 0, 0, 0, 526, 527, 6, 25, 8, 0, 527, 62, 1, 0, 0, 0, 528, 529, 5, 124, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 26, 12, 0, 531, 64, 1, 0, 0, 0, 532, 533, 7, 3, 0, 0, 533, 66, 1, 0, 0, 0, 534, 535, 7, 4, 0, 0, 535, 68, 1, 0, 0, 0, 536, 537, 5, 92, 0, 0, 537, 538, 7, 5, 0, 0, 538, 70, 1, 0, 0, 0, 539, 540, 8, 6, 0, 0, 540, 72, 1, 0, 0, 0, 541, 543, 7, 7, 0, 0, 542, 544, 7, 8, 0, 0, 543, 542, 1, 0, 0, 0, 543, 544, 1, 0, 0, 0, 544, 546, 1, 0, 0, 0, 545, 547, 3, 65, 27, 0, 546, 545, 1, 0, 0, 0, 547, 548, 1, 0, 0, 0, 548, 546, 1, 0, 0, 0, 548, 549, 1, 0, 0, 0, 549, 74, 1, 0, 0, 0, 550, 551, 5, 64, 0, 0, 551, 76, 1, 0, 0, 0, 552, 553, 5, 96, 0, 0, 553, 78, 1, 0, 0, 0, 554, 558, 8, 9, 0, 0, 555, 556, 5, 96, 0, 0, 556, 558, 5, 96, 0, 0, 557, 554, 1, 0, 0, 0, 557, 555, 1, 0, 0, 0, 558, 80, 1, 0, 0, 0, 559, 560, 5, 95, 0, 0, 560, 82, 1, 0, 0, 0, 561, 565, 3, 67, 28, 0, 562, 565, 3, 65, 27, 0, 563, 565, 3, 81, 35, 0, 564, 561, 1, 0, 0, 0, 564, 562, 1, 0, 0, 0, 564, 563, 1, 0, 0, 0, 565, 84, 1, 0, 0, 0, 566, 571, 5, 34, 0, 0, 567, 570, 3, 69, 29, 0, 568, 570, 3, 71, 30, 0, 569, 567, 1, 0, 0, 0, 569, 568, 1, 0, 0, 0, 570, 573, 1, 0, 0, 0, 571, 569, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 572, 574, 1, 0, 0, 0, 573, 571, 1, 0, 0, 0, 574, 596, 5, 34, 0, 0, 575, 576, 5, 34, 0, 0, 576, 577, 5, 34, 0, 0, 577, 578, 5, 34, 0, 0, 578, 582, 1, 0, 0, 0, 579, 581, 8, 1, 0, 0, 580, 579, 1, 0, 0, 0, 581, 584, 1, 0, 0, 0, 582, 583, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 583, 585, 1, 0, 0, 0, 584, 582, 1, 0, 0, 0, 585, 586, 5, 34, 0, 0, 586, 587, 5, 34, 0, 0, 587, 588, 5, 34, 0, 0, 588, 590, 1, 0, 0, 0, 589, 591, 5, 34, 0, 0, 590, 589, 1, 0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 593, 1, 0, 0, 0, 592, 594, 5, 34, 0, 0, 593, 592, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 596, 1, 0, 0, 0, 595, 566, 1, 0, 0, 0, 595, 575, 1, 0, 0, 0, 596, 86, 1, 0, 0, 0, 597, 599, 3, 65, 27, 0, 598, 597, 1, 0, 0, 0, 599, 600, 1, 0, 0, 0, 600, 598, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 88, 1, 0, 0, 0, 602, 604, 3, 65, 27, 0, 603, 602, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 603, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 611, 3, 103, 46, 0, 608, 610, 3, 65, 27, 0, 609, 608, 1, 0, 0, 0, 610, 613, 1, 0, 0, 0, 611, 609, 1, 0, 0, 0, 611, 612, 1, 0, 0, 0, 612, 645, 1, 0, 0, 0, 613, 611, 1, 0, 0, 0, 614, 616, 3, 103, 46, 0, 615, 617, 3, 65, 27, 0, 616, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 645, 1, 0, 0, 0, 620, 622, 3, 65, 27, 0, 621, 620, 1, 0, 0, 0, 622, 623, 1, 0, 0, 0, 623, 621, 1, 0, 0, 0, 623, 624, 1, 0, 0, 0, 624, 632, 1, 0, 0, 0, 625, 629, 3, 103, 46, 0, 626, 628, 3, 65, 27, 0, 627, 626, 1, 0, 0, 0, 628, 631, 1, 0, 0, 0, 629, 627, 1, 0, 0, 0, 629, 630, 1, 0, 0, 0, 630, 633, 1, 0, 0, 0, 631, 629, 1, 0, 0, 0, 632, 625, 1, 0, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 635, 3, 73, 31, 0, 635, 645, 1, 0, 0, 0, 636, 638, 3, 103, 46, 0, 637, 639, 3, 65, 27, 0, 638, 637, 1, 0, 0, 0, 639, 640, 1, 0, 0, 0, 640, 638, 1, 0, 0, 0, 640, 641, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 643, 3, 73, 31, 0, 643, 645, 1, 0, 0, 0, 644, 603, 1, 0, 0, 0, 644, 614, 1, 0, 0, 0, 644, 621, 1, 0, 0, 0, 644, 636, 1, 0, 0, 0, 645, 90, 1, 0, 0, 0, 646, 647, 5, 98, 0, 0, 647, 648, 5, 121, 0, 0, 648, 92, 1, 0, 0, 0, 649, 650, 5, 97, 0, 0, 650, 651, 5, 110, 0, 0, 651, 652, 5, 100, 0, 0, 652, 94, 1, 0, 0, 0, 653, 654, 5, 97, 0, 0, 654, 655, 5, 115, 0, 0, 655, 656, 5, 99, 0, 0, 656, 96, 1, 0, 0, 0, 657, 658, 5, 61, 0, 0, 658, 98, 1, 0, 0, 0, 659, 660, 5, 44, 0, 0, 660, 100, 1, 0, 0, 0, 661, 662, 5, 100, 0, 0, 662, 663, 5, 101, 0, 0, 663, 664, 5, 115, 0, 0, 664, 665, 5, 99, 0, 0, 665, 102, 1, 0, 0, 0, 666, 667, 5, 46, 0, 0, 667, 104, 1, 0, 0, 0, 668, 669, 5, 102, 0, 0, 669, 670, 5, 97, 0, 0, 670, 671, 5, 108, 0, 0, 671, 672, 5, 115, 0, 0, 672, 673, 5, 101, 0, 0, 673, 106, 1, 0, 0, 0, 674, 675, 5, 102, 0, 0, 675, 676, 5, 105, 0, 0, 676, 677, 5, 114, 0, 0, 677, 678, 5, 115, 0, 0, 678, 679, 5, 116, 0, 0, 679, 108, 1, 0, 0, 0, 680, 681, 5, 108, 0, 0, 681, 682, 5, 97, 0, 0, 682, 683, 5, 115, 0, 0, 683, 684, 5, 116, 0, 0, 684, 110, 1, 0, 0, 0, 685, 686, 5, 40, 0, 0, 686, 112, 1, 0, 0, 0, 687, 688, 5, 105, 0, 0, 688, 689, 5, 110, 0, 0, 689, 114, 1, 0, 0, 0, 690, 691, 5, 105, 0, 0, 691, 692, 5, 115, 0, 0, 692, 116, 1, 0, 0, 0, 693, 694, 5, 108, 0, 0, 694, 695, 5, 105, 0, 0, 695, 696, 5, 107, 0, 0, 696, 697, 5, 101, 0, 0, 697, 118, 1, 0, 0, 0, 698, 699, 5, 110, 0, 0, 699, 700, 5, 111, 0, 0, 700, 701, 5, 116, 0, 0, 701, 120, 1, 0, 0, 0, 702, 703, 5, 110, 0, 0, 703, 704, 5, 117, 0, 0, 704, 705, 5, 108, 0, 0, 705, 706, 5, 108, 0, 0, 706, 122, 1, 0, 0, 0, 707, 708, 5, 110, 0, 0, 708, 709, 5, 117, 0, 0, 709, 710, 5, 108, 0, 0, 710, 711, 5, 108, 0, 0, 711, 712, 5, 115, 0, 0, 712, 124, 1, 0, 0, 0, 713, 714, 5, 111, 0, 0, 714, 715, 5, 114, 0, 0, 715, 126, 1, 0, 0, 0, 716, 717, 5, 63, 0, 0, 717, 128, 1, 0, 0, 0, 718, 719, 5, 114, 0, 0, 719, 720, 5, 108, 0, 0, 720, 721, 5, 105, 0, 0, 721, 722, 5, 107, 0, 0, 722, 723, 5, 101, 0, 0, 723, 130, 1, 0, 0, 0, 724, 725, 5, 41, 0, 0, 725, 132, 1, 0, 0, 0, 726, 727, 5, 116, 0, 0, 727, 728, 5, 114, 0, 0, 728, 729, 5, 117, 0, 0, 729, 730, 5, 101, 0, 0, 730, 134, 1, 0, 0, 0, 731, 732, 5, 61, 0, 0, 732, 733, 5, 61, 0, 0, 733, 136, 1, 0, 0, 0, 734, 735, 5, 61, 0, 0, 735, 736, 5, 126, 0, 0, 736, 138, 1, 0, 0, 0, 737, 738, 5, 33, 0, 0, 738, 739, 5, 61, 0, 0, 739, 140, 1, 0, 0, 0, 740, 741, 5, 60, 0, 0, 741, 142, 1, 0, 0, 0, 742, 743, 5, 60, 0, 0, 743, 744, 5, 61, 0, 0, 744, 144, 1, 0, 0, 0, 745, 746, 5, 62, 0, 0, 746, 146, 1, 0, 0, 0, 747, 748, 5, 62, 0, 0, 748, 749, 5, 61, 0, 0, 749, 148, 1, 0, 0, 0, 750, 751, 5, 43, 0, 0, 751, 150, 1, 0, 0, 0, 752, 753, 5, 45, 0, 0, 753, 152, 1, 0, 0, 0, 754, 755, 5, 42, 0, 0, 755, 154, 1, 0, 0, 0, 756, 757, 5, 47, 0, 0, 757, 156, 1, 0, 0, 0, 758, 759, 5, 37, 0, 0, 759, 158, 1, 0, 0, 0, 760, 761, 5, 91, 0, 0, 761, 762, 1, 0, 0, 0, 762, 763, 6, 74, 0, 0, 763, 764, 6, 74, 0, 0, 764, 160, 1, 0, 0, 0, 765, 766, 5, 93, 0, 0, 766, 767, 1, 0, 0, 0, 767, 768, 6, 75, 12, 0, 768, 769, 6, 75, 12, 0, 769, 162, 1, 0, 0, 0, 770, 774, 3, 67, 28, 0, 771, 773, 3, 83, 36, 0, 772, 771, 1, 0, 0, 0, 773, 776, 1, 0, 0, 0, 774, 772, 1, 0, 0, 0, 774, 775, 1, 0, 0, 0, 775, 787, 1, 0, 0, 0, 776, 774, 1, 0, 0, 0, 777, 780, 3, 81, 35, 0, 778, 780, 3, 75, 32, 0, 779, 777, 1, 0, 0, 0, 779, 778, 1, 0, 0, 0, 780, 782, 1, 0, 0, 0, 781, 783, 3, 83, 36, 0, 782, 781, 1, 0, 0, 0, 783, 784, 1, 0, 0, 0, 784, 782, 1, 0, 0, 0, 784, 785, 1, 0, 0, 0, 785, 787, 1, 0, 0, 0, 786, 770, 1, 0, 0, 0, 786, 779, 1, 0, 0, 0, 787, 164, 1, 0, 0, 0, 788, 790, 3, 77, 33, 0, 789, 791, 3, 79, 34, 0, 790, 789, 1, 0, 0, 0, 791, 792, 1, 0, 0, 0, 792, 790, 1, 0, 0, 0, 792, 793, 1, 0, 0, 0, 793, 794, 1, 0, 0, 0, 794, 795, 3, 77, 33, 0, 795, 166, 1, 0, 0, 0, 796, 797, 3, 47, 18, 0, 797, 798, 1, 0, 0, 0, 798, 799, 6, 78, 8, 0, 799, 168, 1, 0, 0, 0, 800, 801, 3, 49, 19, 0, 801, 802, 1, 0, 0, 0, 802, 803, 6, 79, 8, 0, 803, 170, 1, 0, 0, 0, 804, 805, 3, 51, 20, 0, 805, 806, 1, 0, 0, 0, 806, 807, 6, 80, 8, 0, 807, 172, 1, 0, 0, 0, 808, 809, 3, 63, 26, 0, 809, 810, 1, 0, 0, 0, 810, 811, 6, 81, 11, 0, 811, 812, 6, 81, 12, 0, 812, 174, 1, 0, 0, 0, 813, 814, 3, 159, 74, 0, 814, 815, 1, 0, 0, 0, 815, 816, 6, 82, 9, 0, 816, 176, 1, 0, 0, 0, 817, 818, 3, 161, 75, 0, 818, 819, 1, 0, 0, 0, 819, 820, 6, 83, 13, 0, 820, 178, 1, 0, 0, 0, 821, 822, 3, 99, 44, 0, 822, 823, 1, 0, 0, 0, 823, 824, 6, 84, 14, 0, 824, 180, 1, 0, 0, 0, 825, 826, 3, 97, 43, 0, 826, 827, 1, 0, 0, 0, 827, 828, 6, 85, 15, 0, 828, 182, 1, 0, 0, 0, 829, 830, 5, 109, 0, 0, 830, 831, 5, 101, 0, 0, 831, 832, 5, 116, 0, 0, 832, 833, 5, 97, 0, 0, 833, 834, 5, 100, 0, 0, 834, 835, 5, 97, 0, 0, 835, 836, 5, 116, 0, 0, 836, 837, 5, 97, 0, 0, 837, 184, 1, 0, 0, 0, 838, 842, 8, 10, 0, 0, 839, 840, 5, 47, 0, 0, 840, 842, 8, 11, 0, 0, 841, 838, 1, 0, 0, 0, 841, 839, 1, 0, 0, 0, 842, 186, 1, 0, 0, 0, 843, 845, 3, 185, 87, 0, 844, 843, 1, 0, 0, 0, 845, 846, 1, 0, 0, 0, 846, 844, 1, 0, 0, 0, 846, 847, 1, 0, 0, 0, 847, 188, 1, 0, 0, 0, 848, 849, 3, 165, 77, 0, 849, 850, 1, 0, 0, 0, 850, 851, 6, 89, 16, 0, 851, 190, 1, 0, 0, 0, 852, 853, 3, 47, 18, 0, 853, 854, 1, 0, 0, 0, 854, 855, 6, 90, 8, 0, 855, 192, 1, 0, 0, 0, 856, 857, 3, 49, 19, 0, 857, 858, 1, 0, 0, 0, 858, 859, 6, 91, 8, 0, 859, 194, 1, 0, 0, 0, 860, 861, 3, 51, 20, 0, 861, 862, 1, 0, 0, 0, 862, 863, 6, 92, 8, 0, 863, 196, 1, 0, 0, 0, 864, 865, 3, 63, 26, 0, 865, 866, 1, 0, 0, 0, 866, 867, 6, 93, 11, 0, 867, 868, 6, 93, 12, 0, 868, 198, 1, 0, 0, 0, 869, 870, 3, 103, 46, 0, 870, 871, 1, 0, 0, 0, 871, 872, 6, 94, 17, 0, 872, 200, 1, 0, 0, 0, 873, 874, 3, 99, 44, 0, 874, 875, 1, 0, 0, 0, 875, 876, 6, 95, 14, 0, 876, 202, 1, 0, 0, 0, 877, 882, 3, 67, 28, 0, 878, 882, 3, 65, 27, 0, 879, 882, 3, 81, 35, 0, 880, 882, 3, 153, 71, 0, 881, 877, 1, 0, 0, 0, 881, 878, 1, 0, 0, 0, 881, 879, 1, 0, 0, 0, 881, 880, 1, 0, 0, 0, 882, 204, 1, 0, 0, 0, 883, 886, 3, 67, 28, 0, 884, 886, 3, 153, 71, 0, 885, 883, 1, 0, 0, 0, 885, 884, 1, 0, 0, 0, 886, 890, 1, 0, 0, 0, 887, 889, 3, 203, 96, 0, 888, 887, 1, 0, 0, 0, 889, 892, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 903, 1, 0, 0, 0, 892, 890, 1, 0, 0, 0, 893, 896, 3, 81, 35, 0, 894, 896, 3, 75, 32, 0, 895, 893, 1, 0, 0, 0, 895, 894, 1, 0, 0, 0, 896, 898, 1, 0, 0, 0, 897, 899, 3, 203, 96, 0, 898, 897, 1, 0, 0, 0, 899, 900, 1, 0, 0, 0, 900, 898, 1, 0, 0, 0, 900, 901, 1, 0, 0, 0, 901, 903, 1, 0, 0, 0, 902, 885, 1, 0, 0, 0, 902, 895, 1, 0, 0, 0, 903, 206, 1, 0, 0, 0, 904, 905, 3, 205, 97, 0, 905, 906, 1, 0, 0, 0, 906, 907, 6, 98, 18, 0, 907, 208, 1, 0, 0, 0, 908, 909, 3, 165, 77, 0, 909, 910, 1, 0, 0, 0, 910, 911, 6, 99, 16, 0, 911, 210, 1, 0, 0, 0, 912, 913, 3, 47, 18, 0, 913, 914, 1, 0, 0, 0, 914, 915, 6, 100, 8, 0, 915, 212, 1, 0, 0, 0, 916, 917, 3, 49, 19, 0, 917, 918, 1, 0, 0, 0, 918, 919, 6, 101, 8, 0, 919, 214, 1, 0, 0, 0, 920, 921, 3, 51, 20, 0, 921, 922, 1, 0, 0, 0, 922, 923, 6, 102, 8, 0, 923, 216, 1, 0, 0, 0, 924, 925, 3, 63, 26, 0, 925, 926, 1, 0, 0, 0, 926, 927, 6, 103, 11, 0, 927, 928, 6, 103, 12, 0, 928, 218, 1, 0, 0, 0, 929, 930, 3, 97, 43, 0, 930, 931, 1, 0, 0, 0, 931, 932, 6, 104, 15, 0, 932, 220, 1, 0, 0, 0, 933, 934, 3, 99, 44, 0, 934, 935, 1, 0, 0, 0, 935, 936, 6, 105, 14, 0, 936, 222, 1, 0, 0, 0, 937, 938, 3, 103, 46, 0, 938, 939, 1, 0, 0, 0, 939, 940, 6, 106, 17, 0, 940, 224, 1, 0, 0, 0, 941, 942, 5, 97, 0, 0, 942, 943, 5, 115, 0, 0, 943, 226, 1, 0, 0, 0, 944, 945, 3, 165, 77, 0, 945, 946, 1, 0, 0, 0, 946, 947, 6, 108, 16, 0, 947, 228, 1, 0, 0, 0, 948, 949, 3, 205, 97, 0, 949, 950, 1, 0, 0, 0, 950, 951, 6, 109, 18, 0, 951, 230, 1, 0, 0, 0, 952, 953, 3, 47, 18, 0, 953, 954, 1, 0, 0, 0, 954, 955, 6, 110, 8, 0, 955, 232, 1, 0, 0, 0, 956, 957, 3, 49, 19, 0, 957, 958, 1, 0, 0, 0, 958, 959, 6, 111, 8, 0, 959, 234, 1, 0, 0, 0, 960, 961, 3, 51, 20, 0, 961, 962, 1, 0, 0, 0, 962, 963, 6, 112, 8, 0, 963, 236, 1, 0, 0, 0, 964, 965, 3, 63, 26, 0, 965, 966, 1, 0, 0, 0, 966, 967, 6, 113, 11, 0, 967, 968, 6, 113, 12, 0, 968, 238, 1, 0, 0, 0, 969, 970, 3, 159, 74, 0, 970, 971, 1, 0, 0, 0, 971, 972, 6, 114, 9, 0, 972, 973, 6, 114, 19, 0, 973, 240, 1, 0, 0, 0, 974, 975, 5, 111, 0, 0, 975, 976, 5, 110, 0, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 115, 20, 0, 978, 242, 1, 0, 0, 0, 979, 980, 5, 119, 0, 0, 980, 981, 5, 105, 0, 0, 981, 982, 5, 116, 0, 0, 982, 983, 5, 104, 0, 0, 983, 984, 1, 0, 0, 0, 984, 985, 6, 116, 20, 0, 985, 244, 1, 0, 0, 0, 986, 987, 8, 12, 0, 0, 987, 246, 1, 0, 0, 0, 988, 990, 3, 245, 117, 0, 989, 988, 1, 0, 0, 0, 990, 991, 1, 0, 0, 0, 991, 989, 1, 0, 0, 0, 991, 992, 1, 0, 0, 0, 992, 993, 1, 0, 0, 0, 993, 994, 3, 307, 148, 0, 994, 996, 1, 0, 0, 0, 995, 989, 1, 0, 0, 0, 995, 996, 1, 0, 0, 0, 996, 998, 1, 0, 0, 0, 997, 999, 3, 245, 117, 0, 998, 997, 1, 0, 0, 0, 999, 1000, 1, 0, 0, 0, 1000, 998, 1, 0, 0, 0, 1000, 1001, 1, 0, 0, 0, 1001, 248, 1, 0, 0, 0, 1002, 1003, 3, 165, 77, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1005, 6, 119, 16, 0, 1005, 250, 1, 0, 0, 0, 1006, 1007, 3, 247, 118, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 120, 21, 0, 1009, 252, 1, 0, 0, 0, 1010, 1011, 3, 47, 18, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 121, 8, 0, 1013, 254, 1, 0, 0, 0, 1014, 1015, 3, 49, 19, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 122, 8, 0, 1017, 256, 1, 0, 0, 0, 1018, 1019, 3, 51, 20, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 123, 8, 0, 1021, 258, 1, 0, 0, 0, 1022, 1023, 3, 63, 26, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 124, 11, 0, 1025, 1026, 6, 124, 12, 0, 1026, 1027, 6, 124, 12, 0, 1027, 260, 1, 0, 0, 0, 1028, 1029, 3, 97, 43, 0, 1029, 1030, 1, 0, 0, 0, 1030, 1031, 6, 125, 15, 0, 1031, 262, 1, 0, 0, 0, 1032, 1033, 3, 99, 44, 0, 1033, 1034, 1, 0, 0, 0, 1034, 1035, 6, 126, 14, 0, 1035, 264, 1, 0, 0, 0, 1036, 1037, 3, 103, 46, 0, 1037, 1038, 1, 0, 0, 0, 1038, 1039, 6, 127, 17, 0, 1039, 266, 1, 0, 0, 0, 1040, 1041, 3, 243, 116, 0, 1041, 1042, 1, 0, 0, 0, 1042, 1043, 6, 128, 22, 0, 1043, 268, 1, 0, 0, 0, 1044, 1045, 3, 205, 97, 0, 1045, 1046, 1, 0, 0, 0, 1046, 1047, 6, 129, 18, 0, 1047, 270, 1, 0, 0, 0, 1048, 1049, 3, 165, 77, 0, 1049, 1050, 1, 0, 0, 0, 1050, 1051, 6, 130, 16, 0, 1051, 272, 1, 0, 0, 0, 1052, 1053, 3, 47, 18, 0, 1053, 1054, 1, 0, 0, 0, 1054, 1055, 6, 131, 8, 0, 1055, 274, 1, 0, 0, 0, 1056, 1057, 3, 49, 19, 0, 1057, 1058, 1, 0, 0, 0, 1058, 1059, 6, 132, 8, 0, 1059, 276, 1, 0, 0, 0, 1060, 1061, 3, 51, 20, 0, 1061, 1062, 1, 0, 0, 0, 1062, 1063, 6, 133, 8, 0, 1063, 278, 1, 0, 0, 0, 1064, 1065, 3, 63, 26, 0, 1065, 1066, 1, 0, 0, 0, 1066, 1067, 6, 134, 11, 0, 1067, 1068, 6, 134, 12, 0, 1068, 280, 1, 0, 0, 0, 1069, 1070, 3, 103, 46, 0, 1070, 1071, 1, 0, 0, 0, 1071, 1072, 6, 135, 17, 0, 1072, 282, 1, 0, 0, 0, 1073, 1074, 3, 165, 77, 0, 1074, 1075, 1, 0, 0, 0, 1075, 1076, 6, 136, 16, 0, 1076, 284, 1, 0, 0, 0, 1077, 1078, 3, 163, 76, 0, 1078, 1079, 1, 0, 0, 0, 1079, 1080, 6, 137, 23, 0, 1080, 286, 1, 0, 0, 0, 1081, 1082, 3, 47, 18, 0, 1082, 1083, 1, 0, 0, 0, 1083, 1084, 6, 138, 8, 0, 1084, 288, 1, 0, 0, 0, 1085, 1086, 3, 49, 19, 0, 1086, 1087, 1, 0, 0, 0, 1087, 1088, 6, 139, 8, 0, 1088, 290, 1, 0, 0, 0, 1089, 1090, 3, 51, 20, 0, 1090, 1091, 1, 0, 0, 0, 1091, 1092, 6, 140, 8, 0, 1092, 292, 1, 0, 0, 0, 1093, 1094, 3, 63, 26, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 6, 141, 11, 0, 1096, 1097, 6, 141, 12, 0, 1097, 294, 1, 0, 0, 0, 1098, 1099, 5, 105, 0, 0, 1099, 1100, 5, 110, 0, 0, 1100, 1101, 5, 102, 0, 0, 1101, 1102, 5, 111, 0, 0, 1102, 296, 1, 0, 0, 0, 1103, 1104, 5, 102, 0, 0, 1104, 1105, 5, 117, 0, 0, 1105, 1106, 5, 110, 0, 0, 1106, 1107, 5, 99, 0, 0, 1107, 1108, 5, 116, 0, 0, 1108, 1109, 5, 105, 0, 0, 1109, 1110, 5, 111, 0, 0, 1110, 1111, 5, 110, 0, 0, 1111, 1112, 5, 115, 0, 0, 1112, 298, 1, 0, 0, 0, 1113, 1114, 3, 47, 18, 0, 1114, 1115, 1, 0, 0, 0, 1115, 1116, 6, 144, 8, 0, 1116, 300, 1, 0, 0, 0, 1117, 1118, 3, 49, 19, 0, 1118, 1119, 1, 0, 0, 0, 1119, 1120, 6, 145, 8, 0, 1120, 302, 1, 0, 0, 0, 1121, 1122, 3, 51, 20, 0, 1122, 1123, 1, 0, 0, 0, 1123, 1124, 6, 146, 8, 0, 1124, 304, 1, 0, 0, 0, 1125, 1126, 3, 161, 75, 0, 1126, 1127, 1, 0, 0, 0, 1127, 1128, 6, 147, 13, 0, 1128, 1129, 6, 147, 12, 0, 1129, 306, 1, 0, 0, 0, 1130, 1131, 5, 58, 0, 0, 1131, 308, 1, 0, 0, 0, 1132, 1138, 3, 75, 32, 0, 1133, 1138, 3, 65, 27, 0, 1134, 1138, 3, 103, 46, 0, 1135, 1138, 3, 67, 28, 0, 1136, 1138, 3, 81, 35, 0, 1137, 1132, 1, 0, 0, 0, 1137, 1133, 1, 0, 0, 0, 1137, 1134, 1, 0, 0, 0, 1137, 1135, 1, 0, 0, 0, 1137, 1136, 1, 0, 0, 0, 1138, 1139, 1, 0, 0, 0, 1139, 1137, 1, 0, 0, 0, 1139, 1140, 1, 0, 0, 0, 1140, 310, 1, 0, 0, 0, 1141, 1142, 3, 47, 18, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 150, 8, 0, 1144, 312, 1, 0, 0, 0, 1145, 1146, 3, 49, 19, 0, 1146, 1147, 1, 0, 0, 0, 1147, 1148, 6, 151, 8, 0, 1148, 314, 1, 0, 0, 0, 1149, 1150, 3, 51, 20, 0, 1150, 1151, 1, 0, 0, 0, 1151, 1152, 6, 152, 8, 0, 1152, 316, 1, 0, 0, 0, 55, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 463, 473, 477, 480, 489, 491, 502, 543, 548, 557, 564, 569, 571, 582, 590, 593, 595, 600, 605, 611, 618, 623, 629, 632, 640, 644, 774, 779, 784, 786, 792, 841, 846, 881, 885, 890, 895, 900, 902, 991, 995, 1000, 1137, 1139, 24, 5, 2, 0, 5, 4, 0, 5, 6, 0, 5, 1, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 0, 1, 0, 7, 63, 0, 5, 0, 0, 7, 25, 0, 4, 0, 0, 7, 64, 0, 7, 33, 0, 7, 32, 0, 7, 66, 0, 7, 35, 0, 7, 75, 0, 5, 10, 0, 5, 7, 0, 7, 85, 0, 7, 84, 0, 7, 65, 0] \ No newline at end of file +[4, 0, 104, 1147, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 4, 17, 460, 8, 17, 11, 17, 12, 17, 461, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 5, 18, 470, 8, 18, 10, 18, 12, 18, 473, 9, 18, 1, 18, 3, 18, 476, 8, 18, 1, 18, 3, 18, 479, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 488, 8, 19, 10, 19, 12, 19, 491, 9, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 499, 8, 20, 11, 20, 12, 20, 500, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 3, 31, 542, 8, 31, 1, 31, 4, 31, 545, 8, 31, 11, 31, 12, 31, 546, 1, 32, 1, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 3, 34, 556, 8, 34, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 3, 36, 563, 8, 36, 1, 37, 1, 37, 1, 37, 5, 37, 568, 8, 37, 10, 37, 12, 37, 571, 9, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 5, 37, 579, 8, 37, 10, 37, 12, 37, 582, 9, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 3, 37, 589, 8, 37, 1, 37, 3, 37, 592, 8, 37, 3, 37, 594, 8, 37, 1, 38, 4, 38, 597, 8, 38, 11, 38, 12, 38, 598, 1, 39, 4, 39, 602, 8, 39, 11, 39, 12, 39, 603, 1, 39, 1, 39, 5, 39, 608, 8, 39, 10, 39, 12, 39, 611, 9, 39, 1, 39, 1, 39, 4, 39, 615, 8, 39, 11, 39, 12, 39, 616, 1, 39, 4, 39, 620, 8, 39, 11, 39, 12, 39, 621, 1, 39, 1, 39, 5, 39, 626, 8, 39, 10, 39, 12, 39, 629, 9, 39, 3, 39, 631, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 4, 39, 637, 8, 39, 11, 39, 12, 39, 638, 1, 39, 1, 39, 3, 39, 643, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 5, 76, 771, 8, 76, 10, 76, 12, 76, 774, 9, 76, 1, 76, 1, 76, 3, 76, 778, 8, 76, 1, 76, 4, 76, 781, 8, 76, 11, 76, 12, 76, 782, 3, 76, 785, 8, 76, 1, 77, 1, 77, 4, 77, 789, 8, 77, 11, 77, 12, 77, 790, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 3, 88, 842, 8, 88, 1, 89, 4, 89, 845, 8, 89, 11, 89, 12, 89, 846, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 3, 97, 882, 8, 97, 1, 98, 1, 98, 3, 98, 886, 8, 98, 1, 98, 5, 98, 889, 8, 98, 10, 98, 12, 98, 892, 9, 98, 1, 98, 1, 98, 3, 98, 896, 8, 98, 1, 98, 4, 98, 899, 8, 98, 11, 98, 12, 98, 900, 3, 98, 903, 8, 98, 1, 99, 1, 99, 4, 99, 907, 8, 99, 11, 99, 12, 99, 908, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 117, 4, 117, 984, 8, 117, 11, 117, 12, 117, 985, 1, 117, 1, 117, 3, 117, 990, 8, 117, 1, 117, 4, 117, 993, 8, 117, 11, 117, 12, 117, 994, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 148, 4, 148, 1132, 8, 148, 11, 148, 12, 148, 1133, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 2, 489, 580, 0, 152, 11, 1, 13, 2, 15, 3, 17, 4, 19, 5, 21, 6, 23, 7, 25, 8, 27, 9, 29, 10, 31, 11, 33, 12, 35, 13, 37, 14, 39, 15, 41, 16, 43, 17, 45, 18, 47, 19, 49, 20, 51, 21, 53, 0, 55, 0, 57, 22, 59, 23, 61, 24, 63, 25, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 0, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 64, 163, 65, 165, 0, 167, 66, 169, 67, 171, 68, 173, 69, 175, 0, 177, 0, 179, 0, 181, 0, 183, 0, 185, 70, 187, 0, 189, 71, 191, 0, 193, 72, 195, 73, 197, 74, 199, 0, 201, 0, 203, 0, 205, 0, 207, 0, 209, 75, 211, 76, 213, 77, 215, 78, 217, 0, 219, 0, 221, 0, 223, 0, 225, 79, 227, 0, 229, 80, 231, 81, 233, 82, 235, 0, 237, 0, 239, 83, 241, 84, 243, 0, 245, 85, 247, 0, 249, 0, 251, 86, 253, 87, 255, 88, 257, 0, 259, 0, 261, 0, 263, 0, 265, 0, 267, 0, 269, 0, 271, 89, 273, 90, 275, 91, 277, 0, 279, 0, 281, 0, 283, 0, 285, 92, 287, 93, 289, 94, 291, 0, 293, 95, 295, 96, 297, 97, 299, 98, 301, 99, 303, 0, 305, 100, 307, 101, 309, 102, 311, 103, 313, 104, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1175, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 1, 53, 1, 0, 0, 0, 1, 55, 1, 0, 0, 0, 1, 57, 1, 0, 0, 0, 1, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 2, 63, 1, 0, 0, 0, 2, 85, 1, 0, 0, 0, 2, 87, 1, 0, 0, 0, 2, 89, 1, 0, 0, 0, 2, 91, 1, 0, 0, 0, 2, 93, 1, 0, 0, 0, 2, 95, 1, 0, 0, 0, 2, 97, 1, 0, 0, 0, 2, 99, 1, 0, 0, 0, 2, 101, 1, 0, 0, 0, 2, 103, 1, 0, 0, 0, 2, 105, 1, 0, 0, 0, 2, 107, 1, 0, 0, 0, 2, 109, 1, 0, 0, 0, 2, 111, 1, 0, 0, 0, 2, 113, 1, 0, 0, 0, 2, 115, 1, 0, 0, 0, 2, 117, 1, 0, 0, 0, 2, 119, 1, 0, 0, 0, 2, 121, 1, 0, 0, 0, 2, 123, 1, 0, 0, 0, 2, 125, 1, 0, 0, 0, 2, 127, 1, 0, 0, 0, 2, 129, 1, 0, 0, 0, 2, 131, 1, 0, 0, 0, 2, 133, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 2, 137, 1, 0, 0, 0, 2, 139, 1, 0, 0, 0, 2, 141, 1, 0, 0, 0, 2, 143, 1, 0, 0, 0, 2, 145, 1, 0, 0, 0, 2, 147, 1, 0, 0, 0, 2, 149, 1, 0, 0, 0, 2, 151, 1, 0, 0, 0, 2, 153, 1, 0, 0, 0, 2, 155, 1, 0, 0, 0, 2, 157, 1, 0, 0, 0, 2, 159, 1, 0, 0, 0, 2, 161, 1, 0, 0, 0, 2, 163, 1, 0, 0, 0, 2, 167, 1, 0, 0, 0, 2, 169, 1, 0, 0, 0, 2, 171, 1, 0, 0, 0, 2, 173, 1, 0, 0, 0, 3, 175, 1, 0, 0, 0, 3, 177, 1, 0, 0, 0, 3, 179, 1, 0, 0, 0, 3, 181, 1, 0, 0, 0, 3, 183, 1, 0, 0, 0, 3, 185, 1, 0, 0, 0, 3, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 4, 199, 1, 0, 0, 0, 4, 201, 1, 0, 0, 0, 4, 203, 1, 0, 0, 0, 4, 209, 1, 0, 0, 0, 4, 211, 1, 0, 0, 0, 4, 213, 1, 0, 0, 0, 4, 215, 1, 0, 0, 0, 5, 217, 1, 0, 0, 0, 5, 219, 1, 0, 0, 0, 5, 221, 1, 0, 0, 0, 5, 223, 1, 0, 0, 0, 5, 225, 1, 0, 0, 0, 5, 227, 1, 0, 0, 0, 5, 229, 1, 0, 0, 0, 5, 231, 1, 0, 0, 0, 5, 233, 1, 0, 0, 0, 6, 235, 1, 0, 0, 0, 6, 237, 1, 0, 0, 0, 6, 239, 1, 0, 0, 0, 6, 241, 1, 0, 0, 0, 6, 245, 1, 0, 0, 0, 6, 247, 1, 0, 0, 0, 6, 249, 1, 0, 0, 0, 6, 251, 1, 0, 0, 0, 6, 253, 1, 0, 0, 0, 6, 255, 1, 0, 0, 0, 7, 257, 1, 0, 0, 0, 7, 259, 1, 0, 0, 0, 7, 261, 1, 0, 0, 0, 7, 263, 1, 0, 0, 0, 7, 265, 1, 0, 0, 0, 7, 267, 1, 0, 0, 0, 7, 269, 1, 0, 0, 0, 7, 271, 1, 0, 0, 0, 7, 273, 1, 0, 0, 0, 7, 275, 1, 0, 0, 0, 8, 277, 1, 0, 0, 0, 8, 279, 1, 0, 0, 0, 8, 281, 1, 0, 0, 0, 8, 283, 1, 0, 0, 0, 8, 285, 1, 0, 0, 0, 8, 287, 1, 0, 0, 0, 8, 289, 1, 0, 0, 0, 9, 291, 1, 0, 0, 0, 9, 293, 1, 0, 0, 0, 9, 295, 1, 0, 0, 0, 9, 297, 1, 0, 0, 0, 9, 299, 1, 0, 0, 0, 9, 301, 1, 0, 0, 0, 10, 303, 1, 0, 0, 0, 10, 305, 1, 0, 0, 0, 10, 307, 1, 0, 0, 0, 10, 309, 1, 0, 0, 0, 10, 311, 1, 0, 0, 0, 10, 313, 1, 0, 0, 0, 11, 315, 1, 0, 0, 0, 13, 325, 1, 0, 0, 0, 15, 332, 1, 0, 0, 0, 17, 341, 1, 0, 0, 0, 19, 348, 1, 0, 0, 0, 21, 358, 1, 0, 0, 0, 23, 365, 1, 0, 0, 0, 25, 372, 1, 0, 0, 0, 27, 386, 1, 0, 0, 0, 29, 393, 1, 0, 0, 0, 31, 401, 1, 0, 0, 0, 33, 413, 1, 0, 0, 0, 35, 422, 1, 0, 0, 0, 37, 428, 1, 0, 0, 0, 39, 435, 1, 0, 0, 0, 41, 442, 1, 0, 0, 0, 43, 450, 1, 0, 0, 0, 45, 459, 1, 0, 0, 0, 47, 465, 1, 0, 0, 0, 49, 482, 1, 0, 0, 0, 51, 498, 1, 0, 0, 0, 53, 504, 1, 0, 0, 0, 55, 509, 1, 0, 0, 0, 57, 514, 1, 0, 0, 0, 59, 518, 1, 0, 0, 0, 61, 522, 1, 0, 0, 0, 63, 526, 1, 0, 0, 0, 65, 530, 1, 0, 0, 0, 67, 532, 1, 0, 0, 0, 69, 534, 1, 0, 0, 0, 71, 537, 1, 0, 0, 0, 73, 539, 1, 0, 0, 0, 75, 548, 1, 0, 0, 0, 77, 550, 1, 0, 0, 0, 79, 555, 1, 0, 0, 0, 81, 557, 1, 0, 0, 0, 83, 562, 1, 0, 0, 0, 85, 593, 1, 0, 0, 0, 87, 596, 1, 0, 0, 0, 89, 642, 1, 0, 0, 0, 91, 644, 1, 0, 0, 0, 93, 647, 1, 0, 0, 0, 95, 651, 1, 0, 0, 0, 97, 655, 1, 0, 0, 0, 99, 657, 1, 0, 0, 0, 101, 659, 1, 0, 0, 0, 103, 664, 1, 0, 0, 0, 105, 666, 1, 0, 0, 0, 107, 672, 1, 0, 0, 0, 109, 678, 1, 0, 0, 0, 111, 683, 1, 0, 0, 0, 113, 685, 1, 0, 0, 0, 115, 688, 1, 0, 0, 0, 117, 691, 1, 0, 0, 0, 119, 696, 1, 0, 0, 0, 121, 700, 1, 0, 0, 0, 123, 705, 1, 0, 0, 0, 125, 711, 1, 0, 0, 0, 127, 714, 1, 0, 0, 0, 129, 716, 1, 0, 0, 0, 131, 722, 1, 0, 0, 0, 133, 724, 1, 0, 0, 0, 135, 729, 1, 0, 0, 0, 137, 732, 1, 0, 0, 0, 139, 735, 1, 0, 0, 0, 141, 738, 1, 0, 0, 0, 143, 740, 1, 0, 0, 0, 145, 743, 1, 0, 0, 0, 147, 745, 1, 0, 0, 0, 149, 748, 1, 0, 0, 0, 151, 750, 1, 0, 0, 0, 153, 752, 1, 0, 0, 0, 155, 754, 1, 0, 0, 0, 157, 756, 1, 0, 0, 0, 159, 758, 1, 0, 0, 0, 161, 763, 1, 0, 0, 0, 163, 784, 1, 0, 0, 0, 165, 786, 1, 0, 0, 0, 167, 794, 1, 0, 0, 0, 169, 796, 1, 0, 0, 0, 171, 800, 1, 0, 0, 0, 173, 804, 1, 0, 0, 0, 175, 808, 1, 0, 0, 0, 177, 813, 1, 0, 0, 0, 179, 817, 1, 0, 0, 0, 181, 821, 1, 0, 0, 0, 183, 825, 1, 0, 0, 0, 185, 829, 1, 0, 0, 0, 187, 841, 1, 0, 0, 0, 189, 844, 1, 0, 0, 0, 191, 848, 1, 0, 0, 0, 193, 852, 1, 0, 0, 0, 195, 856, 1, 0, 0, 0, 197, 860, 1, 0, 0, 0, 199, 864, 1, 0, 0, 0, 201, 869, 1, 0, 0, 0, 203, 873, 1, 0, 0, 0, 205, 881, 1, 0, 0, 0, 207, 902, 1, 0, 0, 0, 209, 906, 1, 0, 0, 0, 211, 910, 1, 0, 0, 0, 213, 914, 1, 0, 0, 0, 215, 918, 1, 0, 0, 0, 217, 922, 1, 0, 0, 0, 219, 927, 1, 0, 0, 0, 221, 931, 1, 0, 0, 0, 223, 935, 1, 0, 0, 0, 225, 939, 1, 0, 0, 0, 227, 942, 1, 0, 0, 0, 229, 946, 1, 0, 0, 0, 231, 950, 1, 0, 0, 0, 233, 954, 1, 0, 0, 0, 235, 958, 1, 0, 0, 0, 237, 963, 1, 0, 0, 0, 239, 968, 1, 0, 0, 0, 241, 973, 1, 0, 0, 0, 243, 980, 1, 0, 0, 0, 245, 989, 1, 0, 0, 0, 247, 996, 1, 0, 0, 0, 249, 1000, 1, 0, 0, 0, 251, 1004, 1, 0, 0, 0, 253, 1008, 1, 0, 0, 0, 255, 1012, 1, 0, 0, 0, 257, 1016, 1, 0, 0, 0, 259, 1022, 1, 0, 0, 0, 261, 1026, 1, 0, 0, 0, 263, 1030, 1, 0, 0, 0, 265, 1034, 1, 0, 0, 0, 267, 1038, 1, 0, 0, 0, 269, 1042, 1, 0, 0, 0, 271, 1046, 1, 0, 0, 0, 273, 1050, 1, 0, 0, 0, 275, 1054, 1, 0, 0, 0, 277, 1058, 1, 0, 0, 0, 279, 1063, 1, 0, 0, 0, 281, 1067, 1, 0, 0, 0, 283, 1071, 1, 0, 0, 0, 285, 1075, 1, 0, 0, 0, 287, 1079, 1, 0, 0, 0, 289, 1083, 1, 0, 0, 0, 291, 1087, 1, 0, 0, 0, 293, 1092, 1, 0, 0, 0, 295, 1097, 1, 0, 0, 0, 297, 1107, 1, 0, 0, 0, 299, 1111, 1, 0, 0, 0, 301, 1115, 1, 0, 0, 0, 303, 1119, 1, 0, 0, 0, 305, 1124, 1, 0, 0, 0, 307, 1131, 1, 0, 0, 0, 309, 1135, 1, 0, 0, 0, 311, 1139, 1, 0, 0, 0, 313, 1143, 1, 0, 0, 0, 315, 316, 5, 100, 0, 0, 316, 317, 5, 105, 0, 0, 317, 318, 5, 115, 0, 0, 318, 319, 5, 115, 0, 0, 319, 320, 5, 101, 0, 0, 320, 321, 5, 99, 0, 0, 321, 322, 5, 116, 0, 0, 322, 323, 1, 0, 0, 0, 323, 324, 6, 0, 0, 0, 324, 12, 1, 0, 0, 0, 325, 326, 5, 100, 0, 0, 326, 327, 5, 114, 0, 0, 327, 328, 5, 111, 0, 0, 328, 329, 5, 112, 0, 0, 329, 330, 1, 0, 0, 0, 330, 331, 6, 1, 1, 0, 331, 14, 1, 0, 0, 0, 332, 333, 5, 101, 0, 0, 333, 334, 5, 110, 0, 0, 334, 335, 5, 114, 0, 0, 335, 336, 5, 105, 0, 0, 336, 337, 5, 99, 0, 0, 337, 338, 5, 104, 0, 0, 338, 339, 1, 0, 0, 0, 339, 340, 6, 2, 2, 0, 340, 16, 1, 0, 0, 0, 341, 342, 5, 101, 0, 0, 342, 343, 5, 118, 0, 0, 343, 344, 5, 97, 0, 0, 344, 345, 5, 108, 0, 0, 345, 346, 1, 0, 0, 0, 346, 347, 6, 3, 0, 0, 347, 18, 1, 0, 0, 0, 348, 349, 5, 101, 0, 0, 349, 350, 5, 120, 0, 0, 350, 351, 5, 112, 0, 0, 351, 352, 5, 108, 0, 0, 352, 353, 5, 97, 0, 0, 353, 354, 5, 105, 0, 0, 354, 355, 5, 110, 0, 0, 355, 356, 1, 0, 0, 0, 356, 357, 6, 4, 3, 0, 357, 20, 1, 0, 0, 0, 358, 359, 5, 102, 0, 0, 359, 360, 5, 114, 0, 0, 360, 361, 5, 111, 0, 0, 361, 362, 5, 109, 0, 0, 362, 363, 1, 0, 0, 0, 363, 364, 6, 5, 4, 0, 364, 22, 1, 0, 0, 0, 365, 366, 5, 103, 0, 0, 366, 367, 5, 114, 0, 0, 367, 368, 5, 111, 0, 0, 368, 369, 5, 107, 0, 0, 369, 370, 1, 0, 0, 0, 370, 371, 6, 6, 0, 0, 371, 24, 1, 0, 0, 0, 372, 373, 5, 105, 0, 0, 373, 374, 5, 110, 0, 0, 374, 375, 5, 108, 0, 0, 375, 376, 5, 105, 0, 0, 376, 377, 5, 110, 0, 0, 377, 378, 5, 101, 0, 0, 378, 379, 5, 115, 0, 0, 379, 380, 5, 116, 0, 0, 380, 381, 5, 97, 0, 0, 381, 382, 5, 116, 0, 0, 382, 383, 5, 115, 0, 0, 383, 384, 1, 0, 0, 0, 384, 385, 6, 7, 0, 0, 385, 26, 1, 0, 0, 0, 386, 387, 5, 107, 0, 0, 387, 388, 5, 101, 0, 0, 388, 389, 5, 101, 0, 0, 389, 390, 5, 112, 0, 0, 390, 391, 1, 0, 0, 0, 391, 392, 6, 8, 1, 0, 392, 28, 1, 0, 0, 0, 393, 394, 5, 108, 0, 0, 394, 395, 5, 105, 0, 0, 395, 396, 5, 109, 0, 0, 396, 397, 5, 105, 0, 0, 397, 398, 5, 116, 0, 0, 398, 399, 1, 0, 0, 0, 399, 400, 6, 9, 0, 0, 400, 30, 1, 0, 0, 0, 401, 402, 5, 109, 0, 0, 402, 403, 5, 118, 0, 0, 403, 404, 5, 95, 0, 0, 404, 405, 5, 101, 0, 0, 405, 406, 5, 120, 0, 0, 406, 407, 5, 112, 0, 0, 407, 408, 5, 97, 0, 0, 408, 409, 5, 110, 0, 0, 409, 410, 5, 100, 0, 0, 410, 411, 1, 0, 0, 0, 411, 412, 6, 10, 5, 0, 412, 32, 1, 0, 0, 0, 413, 414, 5, 114, 0, 0, 414, 415, 5, 101, 0, 0, 415, 416, 5, 110, 0, 0, 416, 417, 5, 97, 0, 0, 417, 418, 5, 109, 0, 0, 418, 419, 5, 101, 0, 0, 419, 420, 1, 0, 0, 0, 420, 421, 6, 11, 6, 0, 421, 34, 1, 0, 0, 0, 422, 423, 5, 114, 0, 0, 423, 424, 5, 111, 0, 0, 424, 425, 5, 119, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 12, 0, 0, 427, 36, 1, 0, 0, 0, 428, 429, 5, 115, 0, 0, 429, 430, 5, 104, 0, 0, 430, 431, 5, 111, 0, 0, 431, 432, 5, 119, 0, 0, 432, 433, 1, 0, 0, 0, 433, 434, 6, 13, 7, 0, 434, 38, 1, 0, 0, 0, 435, 436, 5, 115, 0, 0, 436, 437, 5, 111, 0, 0, 437, 438, 5, 114, 0, 0, 438, 439, 5, 116, 0, 0, 439, 440, 1, 0, 0, 0, 440, 441, 6, 14, 0, 0, 441, 40, 1, 0, 0, 0, 442, 443, 5, 115, 0, 0, 443, 444, 5, 116, 0, 0, 444, 445, 5, 97, 0, 0, 445, 446, 5, 116, 0, 0, 446, 447, 5, 115, 0, 0, 447, 448, 1, 0, 0, 0, 448, 449, 6, 15, 0, 0, 449, 42, 1, 0, 0, 0, 450, 451, 5, 119, 0, 0, 451, 452, 5, 104, 0, 0, 452, 453, 5, 101, 0, 0, 453, 454, 5, 114, 0, 0, 454, 455, 5, 101, 0, 0, 455, 456, 1, 0, 0, 0, 456, 457, 6, 16, 0, 0, 457, 44, 1, 0, 0, 0, 458, 460, 8, 0, 0, 0, 459, 458, 1, 0, 0, 0, 460, 461, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 461, 462, 1, 0, 0, 0, 462, 463, 1, 0, 0, 0, 463, 464, 6, 17, 0, 0, 464, 46, 1, 0, 0, 0, 465, 466, 5, 47, 0, 0, 466, 467, 5, 47, 0, 0, 467, 471, 1, 0, 0, 0, 468, 470, 8, 1, 0, 0, 469, 468, 1, 0, 0, 0, 470, 473, 1, 0, 0, 0, 471, 469, 1, 0, 0, 0, 471, 472, 1, 0, 0, 0, 472, 475, 1, 0, 0, 0, 473, 471, 1, 0, 0, 0, 474, 476, 5, 13, 0, 0, 475, 474, 1, 0, 0, 0, 475, 476, 1, 0, 0, 0, 476, 478, 1, 0, 0, 0, 477, 479, 5, 10, 0, 0, 478, 477, 1, 0, 0, 0, 478, 479, 1, 0, 0, 0, 479, 480, 1, 0, 0, 0, 480, 481, 6, 18, 8, 0, 481, 48, 1, 0, 0, 0, 482, 483, 5, 47, 0, 0, 483, 484, 5, 42, 0, 0, 484, 489, 1, 0, 0, 0, 485, 488, 3, 49, 19, 0, 486, 488, 9, 0, 0, 0, 487, 485, 1, 0, 0, 0, 487, 486, 1, 0, 0, 0, 488, 491, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 489, 487, 1, 0, 0, 0, 490, 492, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 492, 493, 5, 42, 0, 0, 493, 494, 5, 47, 0, 0, 494, 495, 1, 0, 0, 0, 495, 496, 6, 19, 8, 0, 496, 50, 1, 0, 0, 0, 497, 499, 7, 2, 0, 0, 498, 497, 1, 0, 0, 0, 499, 500, 1, 0, 0, 0, 500, 498, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 20, 8, 0, 503, 52, 1, 0, 0, 0, 504, 505, 3, 159, 74, 0, 505, 506, 1, 0, 0, 0, 506, 507, 6, 21, 9, 0, 507, 508, 6, 21, 10, 0, 508, 54, 1, 0, 0, 0, 509, 510, 3, 63, 26, 0, 510, 511, 1, 0, 0, 0, 511, 512, 6, 22, 11, 0, 512, 513, 6, 22, 12, 0, 513, 56, 1, 0, 0, 0, 514, 515, 3, 51, 20, 0, 515, 516, 1, 0, 0, 0, 516, 517, 6, 23, 8, 0, 517, 58, 1, 0, 0, 0, 518, 519, 3, 47, 18, 0, 519, 520, 1, 0, 0, 0, 520, 521, 6, 24, 8, 0, 521, 60, 1, 0, 0, 0, 522, 523, 3, 49, 19, 0, 523, 524, 1, 0, 0, 0, 524, 525, 6, 25, 8, 0, 525, 62, 1, 0, 0, 0, 526, 527, 5, 124, 0, 0, 527, 528, 1, 0, 0, 0, 528, 529, 6, 26, 12, 0, 529, 64, 1, 0, 0, 0, 530, 531, 7, 3, 0, 0, 531, 66, 1, 0, 0, 0, 532, 533, 7, 4, 0, 0, 533, 68, 1, 0, 0, 0, 534, 535, 5, 92, 0, 0, 535, 536, 7, 5, 0, 0, 536, 70, 1, 0, 0, 0, 537, 538, 8, 6, 0, 0, 538, 72, 1, 0, 0, 0, 539, 541, 7, 7, 0, 0, 540, 542, 7, 8, 0, 0, 541, 540, 1, 0, 0, 0, 541, 542, 1, 0, 0, 0, 542, 544, 1, 0, 0, 0, 543, 545, 3, 65, 27, 0, 544, 543, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 544, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 74, 1, 0, 0, 0, 548, 549, 5, 64, 0, 0, 549, 76, 1, 0, 0, 0, 550, 551, 5, 96, 0, 0, 551, 78, 1, 0, 0, 0, 552, 556, 8, 9, 0, 0, 553, 554, 5, 96, 0, 0, 554, 556, 5, 96, 0, 0, 555, 552, 1, 0, 0, 0, 555, 553, 1, 0, 0, 0, 556, 80, 1, 0, 0, 0, 557, 558, 5, 95, 0, 0, 558, 82, 1, 0, 0, 0, 559, 563, 3, 67, 28, 0, 560, 563, 3, 65, 27, 0, 561, 563, 3, 81, 35, 0, 562, 559, 1, 0, 0, 0, 562, 560, 1, 0, 0, 0, 562, 561, 1, 0, 0, 0, 563, 84, 1, 0, 0, 0, 564, 569, 5, 34, 0, 0, 565, 568, 3, 69, 29, 0, 566, 568, 3, 71, 30, 0, 567, 565, 1, 0, 0, 0, 567, 566, 1, 0, 0, 0, 568, 571, 1, 0, 0, 0, 569, 567, 1, 0, 0, 0, 569, 570, 1, 0, 0, 0, 570, 572, 1, 0, 0, 0, 571, 569, 1, 0, 0, 0, 572, 594, 5, 34, 0, 0, 573, 574, 5, 34, 0, 0, 574, 575, 5, 34, 0, 0, 575, 576, 5, 34, 0, 0, 576, 580, 1, 0, 0, 0, 577, 579, 8, 1, 0, 0, 578, 577, 1, 0, 0, 0, 579, 582, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 580, 578, 1, 0, 0, 0, 581, 583, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 583, 584, 5, 34, 0, 0, 584, 585, 5, 34, 0, 0, 585, 586, 5, 34, 0, 0, 586, 588, 1, 0, 0, 0, 587, 589, 5, 34, 0, 0, 588, 587, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 591, 1, 0, 0, 0, 590, 592, 5, 34, 0, 0, 591, 590, 1, 0, 0, 0, 591, 592, 1, 0, 0, 0, 592, 594, 1, 0, 0, 0, 593, 564, 1, 0, 0, 0, 593, 573, 1, 0, 0, 0, 594, 86, 1, 0, 0, 0, 595, 597, 3, 65, 27, 0, 596, 595, 1, 0, 0, 0, 597, 598, 1, 0, 0, 0, 598, 596, 1, 0, 0, 0, 598, 599, 1, 0, 0, 0, 599, 88, 1, 0, 0, 0, 600, 602, 3, 65, 27, 0, 601, 600, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 601, 1, 0, 0, 0, 603, 604, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 609, 3, 103, 46, 0, 606, 608, 3, 65, 27, 0, 607, 606, 1, 0, 0, 0, 608, 611, 1, 0, 0, 0, 609, 607, 1, 0, 0, 0, 609, 610, 1, 0, 0, 0, 610, 643, 1, 0, 0, 0, 611, 609, 1, 0, 0, 0, 612, 614, 3, 103, 46, 0, 613, 615, 3, 65, 27, 0, 614, 613, 1, 0, 0, 0, 615, 616, 1, 0, 0, 0, 616, 614, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 643, 1, 0, 0, 0, 618, 620, 3, 65, 27, 0, 619, 618, 1, 0, 0, 0, 620, 621, 1, 0, 0, 0, 621, 619, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 630, 1, 0, 0, 0, 623, 627, 3, 103, 46, 0, 624, 626, 3, 65, 27, 0, 625, 624, 1, 0, 0, 0, 626, 629, 1, 0, 0, 0, 627, 625, 1, 0, 0, 0, 627, 628, 1, 0, 0, 0, 628, 631, 1, 0, 0, 0, 629, 627, 1, 0, 0, 0, 630, 623, 1, 0, 0, 0, 630, 631, 1, 0, 0, 0, 631, 632, 1, 0, 0, 0, 632, 633, 3, 73, 31, 0, 633, 643, 1, 0, 0, 0, 634, 636, 3, 103, 46, 0, 635, 637, 3, 65, 27, 0, 636, 635, 1, 0, 0, 0, 637, 638, 1, 0, 0, 0, 638, 636, 1, 0, 0, 0, 638, 639, 1, 0, 0, 0, 639, 640, 1, 0, 0, 0, 640, 641, 3, 73, 31, 0, 641, 643, 1, 0, 0, 0, 642, 601, 1, 0, 0, 0, 642, 612, 1, 0, 0, 0, 642, 619, 1, 0, 0, 0, 642, 634, 1, 0, 0, 0, 643, 90, 1, 0, 0, 0, 644, 645, 5, 98, 0, 0, 645, 646, 5, 121, 0, 0, 646, 92, 1, 0, 0, 0, 647, 648, 5, 97, 0, 0, 648, 649, 5, 110, 0, 0, 649, 650, 5, 100, 0, 0, 650, 94, 1, 0, 0, 0, 651, 652, 5, 97, 0, 0, 652, 653, 5, 115, 0, 0, 653, 654, 5, 99, 0, 0, 654, 96, 1, 0, 0, 0, 655, 656, 5, 61, 0, 0, 656, 98, 1, 0, 0, 0, 657, 658, 5, 44, 0, 0, 658, 100, 1, 0, 0, 0, 659, 660, 5, 100, 0, 0, 660, 661, 5, 101, 0, 0, 661, 662, 5, 115, 0, 0, 662, 663, 5, 99, 0, 0, 663, 102, 1, 0, 0, 0, 664, 665, 5, 46, 0, 0, 665, 104, 1, 0, 0, 0, 666, 667, 5, 102, 0, 0, 667, 668, 5, 97, 0, 0, 668, 669, 5, 108, 0, 0, 669, 670, 5, 115, 0, 0, 670, 671, 5, 101, 0, 0, 671, 106, 1, 0, 0, 0, 672, 673, 5, 102, 0, 0, 673, 674, 5, 105, 0, 0, 674, 675, 5, 114, 0, 0, 675, 676, 5, 115, 0, 0, 676, 677, 5, 116, 0, 0, 677, 108, 1, 0, 0, 0, 678, 679, 5, 108, 0, 0, 679, 680, 5, 97, 0, 0, 680, 681, 5, 115, 0, 0, 681, 682, 5, 116, 0, 0, 682, 110, 1, 0, 0, 0, 683, 684, 5, 40, 0, 0, 684, 112, 1, 0, 0, 0, 685, 686, 5, 105, 0, 0, 686, 687, 5, 110, 0, 0, 687, 114, 1, 0, 0, 0, 688, 689, 5, 105, 0, 0, 689, 690, 5, 115, 0, 0, 690, 116, 1, 0, 0, 0, 691, 692, 5, 108, 0, 0, 692, 693, 5, 105, 0, 0, 693, 694, 5, 107, 0, 0, 694, 695, 5, 101, 0, 0, 695, 118, 1, 0, 0, 0, 696, 697, 5, 110, 0, 0, 697, 698, 5, 111, 0, 0, 698, 699, 5, 116, 0, 0, 699, 120, 1, 0, 0, 0, 700, 701, 5, 110, 0, 0, 701, 702, 5, 117, 0, 0, 702, 703, 5, 108, 0, 0, 703, 704, 5, 108, 0, 0, 704, 122, 1, 0, 0, 0, 705, 706, 5, 110, 0, 0, 706, 707, 5, 117, 0, 0, 707, 708, 5, 108, 0, 0, 708, 709, 5, 108, 0, 0, 709, 710, 5, 115, 0, 0, 710, 124, 1, 0, 0, 0, 711, 712, 5, 111, 0, 0, 712, 713, 5, 114, 0, 0, 713, 126, 1, 0, 0, 0, 714, 715, 5, 63, 0, 0, 715, 128, 1, 0, 0, 0, 716, 717, 5, 114, 0, 0, 717, 718, 5, 108, 0, 0, 718, 719, 5, 105, 0, 0, 719, 720, 5, 107, 0, 0, 720, 721, 5, 101, 0, 0, 721, 130, 1, 0, 0, 0, 722, 723, 5, 41, 0, 0, 723, 132, 1, 0, 0, 0, 724, 725, 5, 116, 0, 0, 725, 726, 5, 114, 0, 0, 726, 727, 5, 117, 0, 0, 727, 728, 5, 101, 0, 0, 728, 134, 1, 0, 0, 0, 729, 730, 5, 61, 0, 0, 730, 731, 5, 61, 0, 0, 731, 136, 1, 0, 0, 0, 732, 733, 5, 61, 0, 0, 733, 734, 5, 126, 0, 0, 734, 138, 1, 0, 0, 0, 735, 736, 5, 33, 0, 0, 736, 737, 5, 61, 0, 0, 737, 140, 1, 0, 0, 0, 738, 739, 5, 60, 0, 0, 739, 142, 1, 0, 0, 0, 740, 741, 5, 60, 0, 0, 741, 742, 5, 61, 0, 0, 742, 144, 1, 0, 0, 0, 743, 744, 5, 62, 0, 0, 744, 146, 1, 0, 0, 0, 745, 746, 5, 62, 0, 0, 746, 747, 5, 61, 0, 0, 747, 148, 1, 0, 0, 0, 748, 749, 5, 43, 0, 0, 749, 150, 1, 0, 0, 0, 750, 751, 5, 45, 0, 0, 751, 152, 1, 0, 0, 0, 752, 753, 5, 42, 0, 0, 753, 154, 1, 0, 0, 0, 754, 755, 5, 47, 0, 0, 755, 156, 1, 0, 0, 0, 756, 757, 5, 37, 0, 0, 757, 158, 1, 0, 0, 0, 758, 759, 5, 91, 0, 0, 759, 760, 1, 0, 0, 0, 760, 761, 6, 74, 0, 0, 761, 762, 6, 74, 0, 0, 762, 160, 1, 0, 0, 0, 763, 764, 5, 93, 0, 0, 764, 765, 1, 0, 0, 0, 765, 766, 6, 75, 12, 0, 766, 767, 6, 75, 12, 0, 767, 162, 1, 0, 0, 0, 768, 772, 3, 67, 28, 0, 769, 771, 3, 83, 36, 0, 770, 769, 1, 0, 0, 0, 771, 774, 1, 0, 0, 0, 772, 770, 1, 0, 0, 0, 772, 773, 1, 0, 0, 0, 773, 785, 1, 0, 0, 0, 774, 772, 1, 0, 0, 0, 775, 778, 3, 81, 35, 0, 776, 778, 3, 75, 32, 0, 777, 775, 1, 0, 0, 0, 777, 776, 1, 0, 0, 0, 778, 780, 1, 0, 0, 0, 779, 781, 3, 83, 36, 0, 780, 779, 1, 0, 0, 0, 781, 782, 1, 0, 0, 0, 782, 780, 1, 0, 0, 0, 782, 783, 1, 0, 0, 0, 783, 785, 1, 0, 0, 0, 784, 768, 1, 0, 0, 0, 784, 777, 1, 0, 0, 0, 785, 164, 1, 0, 0, 0, 786, 788, 3, 77, 33, 0, 787, 789, 3, 79, 34, 0, 788, 787, 1, 0, 0, 0, 789, 790, 1, 0, 0, 0, 790, 788, 1, 0, 0, 0, 790, 791, 1, 0, 0, 0, 791, 792, 1, 0, 0, 0, 792, 793, 3, 77, 33, 0, 793, 166, 1, 0, 0, 0, 794, 795, 3, 165, 77, 0, 795, 168, 1, 0, 0, 0, 796, 797, 3, 47, 18, 0, 797, 798, 1, 0, 0, 0, 798, 799, 6, 79, 8, 0, 799, 170, 1, 0, 0, 0, 800, 801, 3, 49, 19, 0, 801, 802, 1, 0, 0, 0, 802, 803, 6, 80, 8, 0, 803, 172, 1, 0, 0, 0, 804, 805, 3, 51, 20, 0, 805, 806, 1, 0, 0, 0, 806, 807, 6, 81, 8, 0, 807, 174, 1, 0, 0, 0, 808, 809, 3, 63, 26, 0, 809, 810, 1, 0, 0, 0, 810, 811, 6, 82, 11, 0, 811, 812, 6, 82, 12, 0, 812, 176, 1, 0, 0, 0, 813, 814, 3, 159, 74, 0, 814, 815, 1, 0, 0, 0, 815, 816, 6, 83, 9, 0, 816, 178, 1, 0, 0, 0, 817, 818, 3, 161, 75, 0, 818, 819, 1, 0, 0, 0, 819, 820, 6, 84, 13, 0, 820, 180, 1, 0, 0, 0, 821, 822, 3, 99, 44, 0, 822, 823, 1, 0, 0, 0, 823, 824, 6, 85, 14, 0, 824, 182, 1, 0, 0, 0, 825, 826, 3, 97, 43, 0, 826, 827, 1, 0, 0, 0, 827, 828, 6, 86, 15, 0, 828, 184, 1, 0, 0, 0, 829, 830, 5, 109, 0, 0, 830, 831, 5, 101, 0, 0, 831, 832, 5, 116, 0, 0, 832, 833, 5, 97, 0, 0, 833, 834, 5, 100, 0, 0, 834, 835, 5, 97, 0, 0, 835, 836, 5, 116, 0, 0, 836, 837, 5, 97, 0, 0, 837, 186, 1, 0, 0, 0, 838, 842, 8, 10, 0, 0, 839, 840, 5, 47, 0, 0, 840, 842, 8, 11, 0, 0, 841, 838, 1, 0, 0, 0, 841, 839, 1, 0, 0, 0, 842, 188, 1, 0, 0, 0, 843, 845, 3, 187, 88, 0, 844, 843, 1, 0, 0, 0, 845, 846, 1, 0, 0, 0, 846, 844, 1, 0, 0, 0, 846, 847, 1, 0, 0, 0, 847, 190, 1, 0, 0, 0, 848, 849, 3, 167, 78, 0, 849, 850, 1, 0, 0, 0, 850, 851, 6, 90, 16, 0, 851, 192, 1, 0, 0, 0, 852, 853, 3, 47, 18, 0, 853, 854, 1, 0, 0, 0, 854, 855, 6, 91, 8, 0, 855, 194, 1, 0, 0, 0, 856, 857, 3, 49, 19, 0, 857, 858, 1, 0, 0, 0, 858, 859, 6, 92, 8, 0, 859, 196, 1, 0, 0, 0, 860, 861, 3, 51, 20, 0, 861, 862, 1, 0, 0, 0, 862, 863, 6, 93, 8, 0, 863, 198, 1, 0, 0, 0, 864, 865, 3, 63, 26, 0, 865, 866, 1, 0, 0, 0, 866, 867, 6, 94, 11, 0, 867, 868, 6, 94, 12, 0, 868, 200, 1, 0, 0, 0, 869, 870, 3, 103, 46, 0, 870, 871, 1, 0, 0, 0, 871, 872, 6, 95, 17, 0, 872, 202, 1, 0, 0, 0, 873, 874, 3, 99, 44, 0, 874, 875, 1, 0, 0, 0, 875, 876, 6, 96, 14, 0, 876, 204, 1, 0, 0, 0, 877, 882, 3, 67, 28, 0, 878, 882, 3, 65, 27, 0, 879, 882, 3, 81, 35, 0, 880, 882, 3, 153, 71, 0, 881, 877, 1, 0, 0, 0, 881, 878, 1, 0, 0, 0, 881, 879, 1, 0, 0, 0, 881, 880, 1, 0, 0, 0, 882, 206, 1, 0, 0, 0, 883, 886, 3, 67, 28, 0, 884, 886, 3, 153, 71, 0, 885, 883, 1, 0, 0, 0, 885, 884, 1, 0, 0, 0, 886, 890, 1, 0, 0, 0, 887, 889, 3, 205, 97, 0, 888, 887, 1, 0, 0, 0, 889, 892, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 903, 1, 0, 0, 0, 892, 890, 1, 0, 0, 0, 893, 896, 3, 81, 35, 0, 894, 896, 3, 75, 32, 0, 895, 893, 1, 0, 0, 0, 895, 894, 1, 0, 0, 0, 896, 898, 1, 0, 0, 0, 897, 899, 3, 205, 97, 0, 898, 897, 1, 0, 0, 0, 899, 900, 1, 0, 0, 0, 900, 898, 1, 0, 0, 0, 900, 901, 1, 0, 0, 0, 901, 903, 1, 0, 0, 0, 902, 885, 1, 0, 0, 0, 902, 895, 1, 0, 0, 0, 903, 208, 1, 0, 0, 0, 904, 907, 3, 207, 98, 0, 905, 907, 3, 165, 77, 0, 906, 904, 1, 0, 0, 0, 906, 905, 1, 0, 0, 0, 907, 908, 1, 0, 0, 0, 908, 906, 1, 0, 0, 0, 908, 909, 1, 0, 0, 0, 909, 210, 1, 0, 0, 0, 910, 911, 3, 47, 18, 0, 911, 912, 1, 0, 0, 0, 912, 913, 6, 100, 8, 0, 913, 212, 1, 0, 0, 0, 914, 915, 3, 49, 19, 0, 915, 916, 1, 0, 0, 0, 916, 917, 6, 101, 8, 0, 917, 214, 1, 0, 0, 0, 918, 919, 3, 51, 20, 0, 919, 920, 1, 0, 0, 0, 920, 921, 6, 102, 8, 0, 921, 216, 1, 0, 0, 0, 922, 923, 3, 63, 26, 0, 923, 924, 1, 0, 0, 0, 924, 925, 6, 103, 11, 0, 925, 926, 6, 103, 12, 0, 926, 218, 1, 0, 0, 0, 927, 928, 3, 97, 43, 0, 928, 929, 1, 0, 0, 0, 929, 930, 6, 104, 15, 0, 930, 220, 1, 0, 0, 0, 931, 932, 3, 99, 44, 0, 932, 933, 1, 0, 0, 0, 933, 934, 6, 105, 14, 0, 934, 222, 1, 0, 0, 0, 935, 936, 3, 103, 46, 0, 936, 937, 1, 0, 0, 0, 937, 938, 6, 106, 17, 0, 938, 224, 1, 0, 0, 0, 939, 940, 5, 97, 0, 0, 940, 941, 5, 115, 0, 0, 941, 226, 1, 0, 0, 0, 942, 943, 3, 209, 99, 0, 943, 944, 1, 0, 0, 0, 944, 945, 6, 108, 18, 0, 945, 228, 1, 0, 0, 0, 946, 947, 3, 47, 18, 0, 947, 948, 1, 0, 0, 0, 948, 949, 6, 109, 8, 0, 949, 230, 1, 0, 0, 0, 950, 951, 3, 49, 19, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 110, 8, 0, 953, 232, 1, 0, 0, 0, 954, 955, 3, 51, 20, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 111, 8, 0, 957, 234, 1, 0, 0, 0, 958, 959, 3, 63, 26, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 112, 11, 0, 961, 962, 6, 112, 12, 0, 962, 236, 1, 0, 0, 0, 963, 964, 3, 159, 74, 0, 964, 965, 1, 0, 0, 0, 965, 966, 6, 113, 9, 0, 966, 967, 6, 113, 19, 0, 967, 238, 1, 0, 0, 0, 968, 969, 5, 111, 0, 0, 969, 970, 5, 110, 0, 0, 970, 971, 1, 0, 0, 0, 971, 972, 6, 114, 20, 0, 972, 240, 1, 0, 0, 0, 973, 974, 5, 119, 0, 0, 974, 975, 5, 105, 0, 0, 975, 976, 5, 116, 0, 0, 976, 977, 5, 104, 0, 0, 977, 978, 1, 0, 0, 0, 978, 979, 6, 115, 20, 0, 979, 242, 1, 0, 0, 0, 980, 981, 8, 12, 0, 0, 981, 244, 1, 0, 0, 0, 982, 984, 3, 243, 116, 0, 983, 982, 1, 0, 0, 0, 984, 985, 1, 0, 0, 0, 985, 983, 1, 0, 0, 0, 985, 986, 1, 0, 0, 0, 986, 987, 1, 0, 0, 0, 987, 988, 3, 305, 147, 0, 988, 990, 1, 0, 0, 0, 989, 983, 1, 0, 0, 0, 989, 990, 1, 0, 0, 0, 990, 992, 1, 0, 0, 0, 991, 993, 3, 243, 116, 0, 992, 991, 1, 0, 0, 0, 993, 994, 1, 0, 0, 0, 994, 992, 1, 0, 0, 0, 994, 995, 1, 0, 0, 0, 995, 246, 1, 0, 0, 0, 996, 997, 3, 167, 78, 0, 997, 998, 1, 0, 0, 0, 998, 999, 6, 118, 16, 0, 999, 248, 1, 0, 0, 0, 1000, 1001, 3, 245, 117, 0, 1001, 1002, 1, 0, 0, 0, 1002, 1003, 6, 119, 21, 0, 1003, 250, 1, 0, 0, 0, 1004, 1005, 3, 47, 18, 0, 1005, 1006, 1, 0, 0, 0, 1006, 1007, 6, 120, 8, 0, 1007, 252, 1, 0, 0, 0, 1008, 1009, 3, 49, 19, 0, 1009, 1010, 1, 0, 0, 0, 1010, 1011, 6, 121, 8, 0, 1011, 254, 1, 0, 0, 0, 1012, 1013, 3, 51, 20, 0, 1013, 1014, 1, 0, 0, 0, 1014, 1015, 6, 122, 8, 0, 1015, 256, 1, 0, 0, 0, 1016, 1017, 3, 63, 26, 0, 1017, 1018, 1, 0, 0, 0, 1018, 1019, 6, 123, 11, 0, 1019, 1020, 6, 123, 12, 0, 1020, 1021, 6, 123, 12, 0, 1021, 258, 1, 0, 0, 0, 1022, 1023, 3, 97, 43, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 124, 15, 0, 1025, 260, 1, 0, 0, 0, 1026, 1027, 3, 99, 44, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 125, 14, 0, 1029, 262, 1, 0, 0, 0, 1030, 1031, 3, 103, 46, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1033, 6, 126, 17, 0, 1033, 264, 1, 0, 0, 0, 1034, 1035, 3, 241, 115, 0, 1035, 1036, 1, 0, 0, 0, 1036, 1037, 6, 127, 22, 0, 1037, 266, 1, 0, 0, 0, 1038, 1039, 3, 209, 99, 0, 1039, 1040, 1, 0, 0, 0, 1040, 1041, 6, 128, 18, 0, 1041, 268, 1, 0, 0, 0, 1042, 1043, 3, 167, 78, 0, 1043, 1044, 1, 0, 0, 0, 1044, 1045, 6, 129, 16, 0, 1045, 270, 1, 0, 0, 0, 1046, 1047, 3, 47, 18, 0, 1047, 1048, 1, 0, 0, 0, 1048, 1049, 6, 130, 8, 0, 1049, 272, 1, 0, 0, 0, 1050, 1051, 3, 49, 19, 0, 1051, 1052, 1, 0, 0, 0, 1052, 1053, 6, 131, 8, 0, 1053, 274, 1, 0, 0, 0, 1054, 1055, 3, 51, 20, 0, 1055, 1056, 1, 0, 0, 0, 1056, 1057, 6, 132, 8, 0, 1057, 276, 1, 0, 0, 0, 1058, 1059, 3, 63, 26, 0, 1059, 1060, 1, 0, 0, 0, 1060, 1061, 6, 133, 11, 0, 1061, 1062, 6, 133, 12, 0, 1062, 278, 1, 0, 0, 0, 1063, 1064, 3, 103, 46, 0, 1064, 1065, 1, 0, 0, 0, 1065, 1066, 6, 134, 17, 0, 1066, 280, 1, 0, 0, 0, 1067, 1068, 3, 167, 78, 0, 1068, 1069, 1, 0, 0, 0, 1069, 1070, 6, 135, 16, 0, 1070, 282, 1, 0, 0, 0, 1071, 1072, 3, 163, 76, 0, 1072, 1073, 1, 0, 0, 0, 1073, 1074, 6, 136, 23, 0, 1074, 284, 1, 0, 0, 0, 1075, 1076, 3, 47, 18, 0, 1076, 1077, 1, 0, 0, 0, 1077, 1078, 6, 137, 8, 0, 1078, 286, 1, 0, 0, 0, 1079, 1080, 3, 49, 19, 0, 1080, 1081, 1, 0, 0, 0, 1081, 1082, 6, 138, 8, 0, 1082, 288, 1, 0, 0, 0, 1083, 1084, 3, 51, 20, 0, 1084, 1085, 1, 0, 0, 0, 1085, 1086, 6, 139, 8, 0, 1086, 290, 1, 0, 0, 0, 1087, 1088, 3, 63, 26, 0, 1088, 1089, 1, 0, 0, 0, 1089, 1090, 6, 140, 11, 0, 1090, 1091, 6, 140, 12, 0, 1091, 292, 1, 0, 0, 0, 1092, 1093, 5, 105, 0, 0, 1093, 1094, 5, 110, 0, 0, 1094, 1095, 5, 102, 0, 0, 1095, 1096, 5, 111, 0, 0, 1096, 294, 1, 0, 0, 0, 1097, 1098, 5, 102, 0, 0, 1098, 1099, 5, 117, 0, 0, 1099, 1100, 5, 110, 0, 0, 1100, 1101, 5, 99, 0, 0, 1101, 1102, 5, 116, 0, 0, 1102, 1103, 5, 105, 0, 0, 1103, 1104, 5, 111, 0, 0, 1104, 1105, 5, 110, 0, 0, 1105, 1106, 5, 115, 0, 0, 1106, 296, 1, 0, 0, 0, 1107, 1108, 3, 47, 18, 0, 1108, 1109, 1, 0, 0, 0, 1109, 1110, 6, 143, 8, 0, 1110, 298, 1, 0, 0, 0, 1111, 1112, 3, 49, 19, 0, 1112, 1113, 1, 0, 0, 0, 1113, 1114, 6, 144, 8, 0, 1114, 300, 1, 0, 0, 0, 1115, 1116, 3, 51, 20, 0, 1116, 1117, 1, 0, 0, 0, 1117, 1118, 6, 145, 8, 0, 1118, 302, 1, 0, 0, 0, 1119, 1120, 3, 161, 75, 0, 1120, 1121, 1, 0, 0, 0, 1121, 1122, 6, 146, 13, 0, 1122, 1123, 6, 146, 12, 0, 1123, 304, 1, 0, 0, 0, 1124, 1125, 5, 58, 0, 0, 1125, 306, 1, 0, 0, 0, 1126, 1132, 3, 75, 32, 0, 1127, 1132, 3, 65, 27, 0, 1128, 1132, 3, 103, 46, 0, 1129, 1132, 3, 67, 28, 0, 1130, 1132, 3, 81, 35, 0, 1131, 1126, 1, 0, 0, 0, 1131, 1127, 1, 0, 0, 0, 1131, 1128, 1, 0, 0, 0, 1131, 1129, 1, 0, 0, 0, 1131, 1130, 1, 0, 0, 0, 1132, 1133, 1, 0, 0, 0, 1133, 1131, 1, 0, 0, 0, 1133, 1134, 1, 0, 0, 0, 1134, 308, 1, 0, 0, 0, 1135, 1136, 3, 47, 18, 0, 1136, 1137, 1, 0, 0, 0, 1137, 1138, 6, 149, 8, 0, 1138, 310, 1, 0, 0, 0, 1139, 1140, 3, 49, 19, 0, 1140, 1141, 1, 0, 0, 0, 1141, 1142, 6, 150, 8, 0, 1142, 312, 1, 0, 0, 0, 1143, 1144, 3, 51, 20, 0, 1144, 1145, 1, 0, 0, 0, 1145, 1146, 6, 151, 8, 0, 1146, 314, 1, 0, 0, 0, 57, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 461, 471, 475, 478, 487, 489, 500, 541, 546, 555, 562, 567, 569, 580, 588, 591, 593, 598, 603, 609, 616, 621, 627, 630, 638, 642, 772, 777, 782, 784, 790, 841, 846, 881, 885, 890, 895, 900, 902, 906, 908, 985, 989, 994, 1131, 1133, 24, 5, 2, 0, 5, 4, 0, 5, 6, 0, 5, 1, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 0, 1, 0, 7, 63, 0, 5, 0, 0, 7, 25, 0, 4, 0, 0, 7, 64, 0, 7, 33, 0, 7, 32, 0, 7, 66, 0, 7, 35, 0, 7, 75, 0, 5, 10, 0, 5, 7, 0, 7, 85, 0, 7, 84, 0, 7, 65, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index 0a6e512e3b318..02ba5f7caacde 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -28,7 +28,7 @@ public class EsqlBaseLexer extends Lexer { GTE=57, PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, OPENING_BRACKET=63, CLOSING_BRACKET=64, UNQUOTED_IDENTIFIER=65, QUOTED_IDENTIFIER=66, EXPR_LINE_COMMENT=67, EXPR_MULTILINE_COMMENT=68, EXPR_WS=69, METADATA=70, FROM_UNQUOTED_IDENTIFIER=71, - FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, UNQUOTED_ID_PATTERN=75, + FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, ID_PATTERN=75, PROJECT_LINE_COMMENT=76, PROJECT_MULTILINE_COMMENT=77, PROJECT_WS=78, AS=79, RENAME_LINE_COMMENT=80, RENAME_MULTILINE_COMMENT=81, RENAME_WS=82, ON=83, WITH=84, ENRICH_POLICY_NAME=85, ENRICH_LINE_COMMENT=86, ENRICH_MULTILINE_COMMENT=87, @@ -63,26 +63,25 @@ private static String[] makeRuleNames() { "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", - "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", - "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", "FROM_COMMA", - "FROM_ASSIGN", "METADATA", "FROM_UNQUOTED_IDENTIFIER_PART", "FROM_UNQUOTED_IDENTIFIER", - "FROM_QUOTED_IDENTIFIER", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", - "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", "UNQUOTED_ID_BODY_WITH_PATTERN", - "UNQUOTED_ID_PATTERN", "PROJECT_UNQUOTED_IDENTIFIER", "PROJECT_QUOTED_IDENTIFIER", + "QUOTED_ID", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", + "EXPR_WS", "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", + "FROM_COMMA", "FROM_ASSIGN", "METADATA", "FROM_UNQUOTED_IDENTIFIER_PART", + "FROM_UNQUOTED_IDENTIFIER", "FROM_QUOTED_IDENTIFIER", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", + "UNQUOTED_ID_BODY_WITH_PATTERN", "UNQUOTED_ID_PATTERN", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "RENAME_PIPE", - "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", "AS", "RENAME_QUOTED_IDENTIFIER", - "RENAME_UNQUOTED_IDENTIFIER", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", - "RENAME_WS", "ENRICH_PIPE", "ENRICH_OPENING_BRACKET", "ON", "WITH", "ENRICH_POLICY_NAME_BODY", - "ENRICH_POLICY_NAME", "ENRICH_QUOTED_IDENTIFIER", "ENRICH_MODE_UNQUOTED_VALUE", - "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_PIPE", - "ENRICH_FIELD_ASSIGN", "ENRICH_FIELD_COMMA", "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", - "ENRICH_FIELD_UNQUOTED_IDENTIFIER", "ENRICH_FIELD_QUOTED_IDENTIFIER", - "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", - "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", "MVEXPAND_UNQUOTED_IDENTIFIER", - "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", - "SHOW_PIPE", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", - "SHOW_WS", "SETTING_CLOSING_BRACKET", "COLON", "SETTING", "SETTING_LINE_COMMENT", - "SETTTING_MULTILINE_COMMENT", "SETTING_WS" + "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", "AS", "RENAME_ID_PATTERN", + "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ENRICH_PIPE", + "ENRICH_OPENING_BRACKET", "ON", "WITH", "ENRICH_POLICY_NAME_BODY", "ENRICH_POLICY_NAME", + "ENRICH_QUOTED_IDENTIFIER", "ENRICH_MODE_UNQUOTED_VALUE", "ENRICH_LINE_COMMENT", + "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_PIPE", "ENRICH_FIELD_ASSIGN", + "ENRICH_FIELD_COMMA", "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", "ENRICH_FIELD_ID_PATTERN", + "ENRICH_FIELD_QUOTED_IDENTIFIER", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", + "ENRICH_FIELD_WS", "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", + "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", + "MVEXPAND_WS", "SHOW_PIPE", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS", "SETTING_CLOSING_BRACKET", "COLON", + "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS" }; } public static final String[] ruleNames = makeRuleNames(); @@ -117,7 +116,7 @@ private static String[] makeSymbolicNames() { "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "METADATA", "FROM_UNQUOTED_IDENTIFIER", "FROM_LINE_COMMENT", - "FROM_MULTILINE_COMMENT", "FROM_WS", "UNQUOTED_ID_PATTERN", "PROJECT_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", @@ -187,7 +186,7 @@ public EsqlBaseLexer(CharStream input) { public ATN getATN() { return _ATN; } public static final String _serializedATN = - "\u0004\u0000h\u0481\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0004\u0000h\u047b\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002"+ @@ -227,143 +226,142 @@ public EsqlBaseLexer(CharStream input) { "\u008d\u0007\u008d\u0002\u008e\u0007\u008e\u0002\u008f\u0007\u008f\u0002"+ "\u0090\u0007\u0090\u0002\u0091\u0007\u0091\u0002\u0092\u0007\u0092\u0002"+ "\u0093\u0007\u0093\u0002\u0094\u0007\u0094\u0002\u0095\u0007\u0095\u0002"+ - "\u0096\u0007\u0096\u0002\u0097\u0007\u0097\u0002\u0098\u0007\u0098\u0001"+ + "\u0096\u0007\u0096\u0002\u0097\u0007\u0097\u0001\u0000\u0001\u0000\u0001"+ "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ - "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001"+ + "\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ - "\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ - "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ - "\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ - "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b"+ - "\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b"+ + "\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b"+ "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ - "\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+ + "\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001"+ + "\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e"+ + "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ - "\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ - "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0004\u0011"+ - "\u01ce\b\u0011\u000b\u0011\f\u0011\u01cf\u0001\u0011\u0001\u0011\u0001"+ - "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0005\u0012\u01d8\b\u0012\n"+ - "\u0012\f\u0012\u01db\t\u0012\u0001\u0012\u0003\u0012\u01de\b\u0012\u0001"+ - "\u0012\u0003\u0012\u01e1\b\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001"+ - "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u01ea\b\u0013\n"+ - "\u0013\f\u0013\u01ed\t\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"+ - "\u0013\u0001\u0013\u0001\u0014\u0004\u0014\u01f5\b\u0014\u000b\u0014\f"+ - "\u0014\u01f6\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015"+ - "\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016"+ - "\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018"+ - "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019"+ - "\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b"+ - "\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0001\u001d"+ - "\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0003\u001f\u0220\b\u001f"+ - "\u0001\u001f\u0004\u001f\u0223\b\u001f\u000b\u001f\f\u001f\u0224\u0001"+ - " \u0001 \u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0003\"\u022e\b\"\u0001"+ - "#\u0001#\u0001$\u0001$\u0001$\u0003$\u0235\b$\u0001%\u0001%\u0001%\u0005"+ - "%\u023a\b%\n%\f%\u023d\t%\u0001%\u0001%\u0001%\u0001%\u0001%\u0001%\u0005"+ - "%\u0245\b%\n%\f%\u0248\t%\u0001%\u0001%\u0001%\u0001%\u0001%\u0003%\u024f"+ - "\b%\u0001%\u0003%\u0252\b%\u0003%\u0254\b%\u0001&\u0004&\u0257\b&\u000b"+ - "&\f&\u0258\u0001\'\u0004\'\u025c\b\'\u000b\'\f\'\u025d\u0001\'\u0001\'"+ - "\u0005\'\u0262\b\'\n\'\f\'\u0265\t\'\u0001\'\u0001\'\u0004\'\u0269\b\'"+ - "\u000b\'\f\'\u026a\u0001\'\u0004\'\u026e\b\'\u000b\'\f\'\u026f\u0001\'"+ - "\u0001\'\u0005\'\u0274\b\'\n\'\f\'\u0277\t\'\u0003\'\u0279\b\'\u0001\'"+ - "\u0001\'\u0001\'\u0001\'\u0004\'\u027f\b\'\u000b\'\f\'\u0280\u0001\'\u0001"+ - "\'\u0003\'\u0285\b\'\u0001(\u0001(\u0001(\u0001)\u0001)\u0001)\u0001)"+ - "\u0001*\u0001*\u0001*\u0001*\u0001+\u0001+\u0001,\u0001,\u0001-\u0001"+ - "-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0001/\u0001/\u0001"+ - "/\u0001/\u00010\u00010\u00010\u00010\u00010\u00010\u00011\u00011\u0001"+ - "1\u00011\u00011\u00012\u00012\u00013\u00013\u00013\u00014\u00014\u0001"+ - "4\u00015\u00015\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u0001"+ - "7\u00017\u00017\u00017\u00017\u00018\u00018\u00018\u00018\u00018\u0001"+ - "8\u00019\u00019\u00019\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001"+ - ";\u0001;\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0001=\u0001>\u0001"+ - ">\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001@\u0001A\u0001A\u0001"+ - "B\u0001B\u0001B\u0001C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001"+ - "F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001J\u0001J\u0001"+ - "J\u0001J\u0001J\u0001K\u0001K\u0001K\u0001K\u0001K\u0001L\u0001L\u0005"+ - "L\u0305\bL\nL\fL\u0308\tL\u0001L\u0001L\u0003L\u030c\bL\u0001L\u0004L"+ - "\u030f\bL\u000bL\fL\u0310\u0003L\u0313\bL\u0001M\u0001M\u0004M\u0317\b"+ - "M\u000bM\fM\u0318\u0001M\u0001M\u0001N\u0001N\u0001N\u0001N\u0001O\u0001"+ - "O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001Q\u0001"+ - "Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001"+ - "T\u0001T\u0001T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001"+ - "V\u0001V\u0001V\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0003"+ - "W\u034a\bW\u0001X\u0004X\u034d\bX\u000bX\fX\u034e\u0001Y\u0001Y\u0001"+ - "Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001"+ - "\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001]\u0001^\u0001"+ - "^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001`\u0001"+ - "`\u0003`\u0372\b`\u0001a\u0001a\u0003a\u0376\ba\u0001a\u0005a\u0379\b"+ - "a\na\fa\u037c\ta\u0001a\u0001a\u0003a\u0380\ba\u0001a\u0004a\u0383\ba"+ - "\u000ba\fa\u0384\u0003a\u0387\ba\u0001b\u0001b\u0001b\u0001b\u0001c\u0001"+ - "c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001"+ - "e\u0001f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001g\u0001g\u0001g\u0001"+ - "h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001"+ - "j\u0001j\u0001k\u0001k\u0001k\u0001l\u0001l\u0001l\u0001l\u0001m\u0001"+ - "m\u0001m\u0001m\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001"+ - "o\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001q\u0001"+ - "r\u0001r\u0001r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001s\u0001"+ - "t\u0001t\u0001t\u0001t\u0001t\u0001t\u0001t\u0001u\u0001u\u0001v\u0004"+ - "v\u03de\bv\u000bv\fv\u03df\u0001v\u0001v\u0003v\u03e4\bv\u0001v\u0004"+ - "v\u03e7\bv\u000bv\fv\u03e8\u0001w\u0001w\u0001w\u0001w\u0001x\u0001x\u0001"+ - "x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001"+ - "{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001|\u0001|\u0001|\u0001|\u0001"+ - "}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f"+ - "\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080"+ - "\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0082\u0001\u0082"+ - "\u0001\u0082\u0001\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083"+ - "\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085"+ - "\u0001\u0085\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086"+ - "\u0001\u0086\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088"+ - "\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089"+ - "\u0001\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b"+ - "\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c"+ - "\u0001\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d"+ - "\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f"+ - "\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f"+ - "\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090"+ - "\u0001\u0090\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092"+ - "\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093"+ - "\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095"+ - "\u0001\u0095\u0001\u0095\u0001\u0095\u0004\u0095\u0472\b\u0095\u000b\u0095"+ - "\f\u0095\u0473\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097"+ - "\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098"+ - "\u0001\u0098\u0002\u01eb\u0246\u0000\u0099\u000b\u0001\r\u0002\u000f\u0003"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ + "\u0001\u0010\u0001\u0010\u0001\u0011\u0004\u0011\u01cc\b\u0011\u000b\u0011"+ + "\f\u0011\u01cd\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0012\u0005\u0012\u01d6\b\u0012\n\u0012\f\u0012\u01d9\t\u0012\u0001"+ + "\u0012\u0003\u0012\u01dc\b\u0012\u0001\u0012\u0003\u0012\u01df\b\u0012"+ + "\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+ + "\u0001\u0013\u0005\u0013\u01e8\b\u0013\n\u0013\f\u0013\u01eb\t\u0013\u0001"+ + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0004"+ + "\u0014\u01f3\b\u0014\u000b\u0014\f\u0014\u01f4\u0001\u0014\u0001\u0014"+ + "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0016"+ + "\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017"+ + "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018"+ + "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a"+ + "\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c"+ + "\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001f"+ + "\u0001\u001f\u0003\u001f\u021e\b\u001f\u0001\u001f\u0004\u001f\u0221\b"+ + "\u001f\u000b\u001f\f\u001f\u0222\u0001 \u0001 \u0001!\u0001!\u0001\"\u0001"+ + "\"\u0001\"\u0003\"\u022c\b\"\u0001#\u0001#\u0001$\u0001$\u0001$\u0003"+ + "$\u0233\b$\u0001%\u0001%\u0001%\u0005%\u0238\b%\n%\f%\u023b\t%\u0001%"+ + "\u0001%\u0001%\u0001%\u0001%\u0001%\u0005%\u0243\b%\n%\f%\u0246\t%\u0001"+ + "%\u0001%\u0001%\u0001%\u0001%\u0003%\u024d\b%\u0001%\u0003%\u0250\b%\u0003"+ + "%\u0252\b%\u0001&\u0004&\u0255\b&\u000b&\f&\u0256\u0001\'\u0004\'\u025a"+ + "\b\'\u000b\'\f\'\u025b\u0001\'\u0001\'\u0005\'\u0260\b\'\n\'\f\'\u0263"+ + "\t\'\u0001\'\u0001\'\u0004\'\u0267\b\'\u000b\'\f\'\u0268\u0001\'\u0004"+ + "\'\u026c\b\'\u000b\'\f\'\u026d\u0001\'\u0001\'\u0005\'\u0272\b\'\n\'\f"+ + "\'\u0275\t\'\u0003\'\u0277\b\'\u0001\'\u0001\'\u0001\'\u0001\'\u0004\'"+ + "\u027d\b\'\u000b\'\f\'\u027e\u0001\'\u0001\'\u0003\'\u0283\b\'\u0001("+ + "\u0001(\u0001(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001"+ + "*\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001-\u0001"+ + ".\u0001.\u0001/\u0001/\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u0001"+ + "0\u00010\u00010\u00010\u00011\u00011\u00011\u00011\u00011\u00012\u0001"+ + "2\u00013\u00013\u00013\u00014\u00014\u00014\u00015\u00015\u00015\u0001"+ + "5\u00015\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u0001"+ + "7\u00018\u00018\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001"+ + ":\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001<\u0001"+ + "=\u0001=\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001"+ + "?\u0001@\u0001@\u0001@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001C\u0001"+ + "C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001"+ + "H\u0001H\u0001I\u0001I\u0001J\u0001J\u0001J\u0001J\u0001J\u0001K\u0001"+ + "K\u0001K\u0001K\u0001K\u0001L\u0001L\u0005L\u0303\bL\nL\fL\u0306\tL\u0001"+ + "L\u0001L\u0003L\u030a\bL\u0001L\u0004L\u030d\bL\u000bL\fL\u030e\u0003"+ + "L\u0311\bL\u0001M\u0001M\u0004M\u0315\bM\u000bM\fM\u0316\u0001M\u0001"+ + "M\u0001N\u0001N\u0001O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001"+ + "P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001R\u0001"+ + "S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001T\u0001U\u0001U\u0001"+ + "U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001"+ + "W\u0001W\u0001W\u0001W\u0001W\u0001X\u0001X\u0001X\u0003X\u034a\bX\u0001"+ + "Y\u0004Y\u034d\bY\u000bY\fY\u034e\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001"+ + "[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001"+ + "]\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001"+ + "`\u0001`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001a\u0003a\u0372\ba\u0001"+ + "b\u0001b\u0003b\u0376\bb\u0001b\u0005b\u0379\bb\nb\fb\u037c\tb\u0001b"+ + "\u0001b\u0003b\u0380\bb\u0001b\u0004b\u0383\bb\u000bb\fb\u0384\u0003b"+ + "\u0387\bb\u0001c\u0001c\u0004c\u038b\bc\u000bc\fc\u038c\u0001d\u0001d"+ + "\u0001d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001"+ + "f\u0001g\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001"+ + "i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001"+ + "k\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001m\u0001n\u0001"+ + "n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001"+ + "p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001"+ + "r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001s\u0001s\u0001s\u0001t\u0001"+ + "t\u0001u\u0004u\u03d8\bu\u000bu\fu\u03d9\u0001u\u0001u\u0003u\u03de\b"+ + "u\u0001u\u0004u\u03e1\bu\u000bu\fu\u03e2\u0001v\u0001v\u0001v\u0001v\u0001"+ + "w\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+ + "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+ + "{\u0001{\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001"+ + "~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f"+ + "\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081"+ + "\u0001\u0081\u0001\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082"+ + "\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084"+ + "\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085"+ + "\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087"+ + "\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0088"+ + "\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a"+ + "\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008b"+ + "\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c"+ + "\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e"+ + "\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e"+ + "\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f"+ + "\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091"+ + "\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092"+ + "\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094"+ + "\u0001\u0094\u0001\u0094\u0001\u0094\u0004\u0094\u046c\b\u0094\u000b\u0094"+ + "\f\u0094\u046d\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0096"+ + "\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001\u0097\u0001\u0097"+ + "\u0001\u0097\u0002\u01e9\u0244\u0000\u0098\u000b\u0001\r\u0002\u000f\u0003"+ "\u0011\u0004\u0013\u0005\u0015\u0006\u0017\u0007\u0019\b\u001b\t\u001d"+ "\n\u001f\u000b!\f#\r%\u000e\'\u000f)\u0010+\u0011-\u0012/\u00131\u0014"+ "3\u00155\u00007\u00009\u0016;\u0017=\u0018?\u0019A\u0000C\u0000E\u0000"+ "G\u0000I\u0000K\u0000M\u0000O\u0000Q\u0000S\u0000U\u001aW\u001bY\u001c"+ "[\u001d]\u001e_\u001fa c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f/\u00810\u0083"+ "1\u00852\u00873\u00894\u008b5\u008d6\u008f7\u00918\u00939\u0095:\u0097"+ - ";\u0099<\u009b=\u009d>\u009f?\u00a1@\u00a3A\u00a5B\u00a7C\u00a9D\u00ab"+ - "E\u00ad\u0000\u00af\u0000\u00b1\u0000\u00b3\u0000\u00b5\u0000\u00b7F\u00b9"+ - "\u0000\u00bbG\u00bd\u0000\u00bfH\u00c1I\u00c3J\u00c5\u0000\u00c7\u0000"+ - "\u00c9\u0000\u00cb\u0000\u00cdK\u00cf\u0000\u00d1\u0000\u00d3L\u00d5M"+ + ";\u0099<\u009b=\u009d>\u009f?\u00a1@\u00a3A\u00a5\u0000\u00a7B\u00a9C"+ + "\u00abD\u00adE\u00af\u0000\u00b1\u0000\u00b3\u0000\u00b5\u0000\u00b7\u0000"+ + "\u00b9F\u00bb\u0000\u00bdG\u00bf\u0000\u00c1H\u00c3I\u00c5J\u00c7\u0000"+ + "\u00c9\u0000\u00cb\u0000\u00cd\u0000\u00cf\u0000\u00d1K\u00d3L\u00d5M"+ "\u00d7N\u00d9\u0000\u00db\u0000\u00dd\u0000\u00df\u0000\u00e1O\u00e3\u0000"+ - "\u00e5\u0000\u00e7P\u00e9Q\u00ebR\u00ed\u0000\u00ef\u0000\u00f1S\u00f3"+ - "T\u00f5\u0000\u00f7U\u00f9\u0000\u00fb\u0000\u00fdV\u00ffW\u0101X\u0103"+ + "\u00e5P\u00e7Q\u00e9R\u00eb\u0000\u00ed\u0000\u00efS\u00f1T\u00f3\u0000"+ + "\u00f5U\u00f7\u0000\u00f9\u0000\u00fbV\u00fdW\u00ffX\u0101\u0000\u0103"+ "\u0000\u0105\u0000\u0107\u0000\u0109\u0000\u010b\u0000\u010d\u0000\u010f"+ - "\u0000\u0111Y\u0113Z\u0115[\u0117\u0000\u0119\u0000\u011b\u0000\u011d"+ - "\u0000\u011f\\\u0121]\u0123^\u0125\u0000\u0127_\u0129`\u012ba\u012db\u012f"+ - "c\u0131\u0000\u0133d\u0135e\u0137f\u0139g\u013bh\u000b\u0000\u0001\u0002"+ - "\u0003\u0004\u0005\u0006\u0007\b\t\n\r\u0006\u0000\t\n\r\r //[[]]\u0002"+ - "\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002\u0000AZaz\u0005"+ - "\u0000\"\"\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000EEee\u0002"+ - "\u0000++--\u0001\u0000``\n\u0000\t\n\r\r ,,//==[[]]``||\u0002\u0000*"+ - "*//\u000b\u0000\t\n\r\r \"#,,//::<<>?\\\\||\u049d\u0000\u000b\u0001\u0000"+ - "\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+ - "\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+ - "\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+ - "\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+ - "\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+ - "\u0000\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000"+ - "%\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001"+ + "Y\u0111Z\u0113[\u0115\u0000\u0117\u0000\u0119\u0000\u011b\u0000\u011d"+ + "\\\u011f]\u0121^\u0123\u0000\u0125_\u0127`\u0129a\u012bb\u012dc\u012f"+ + "\u0000\u0131d\u0133e\u0135f\u0137g\u0139h\u000b\u0000\u0001\u0002\u0003"+ + "\u0004\u0005\u0006\u0007\b\t\n\r\u0006\u0000\t\n\r\r //[[]]\u0002\u0000"+ + "\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002\u0000AZaz\u0005\u0000"+ + "\"\"\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000EEee\u0002\u0000"+ + "++--\u0001\u0000``\n\u0000\t\n\r\r ,,//==[[]]``||\u0002\u0000**//\u000b"+ + "\u0000\t\n\r\r \"#,,//::<<>?\\\\||\u0497\u0000\u000b\u0001\u0000\u0000"+ + "\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000\u0000"+ + "\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000"+ + "\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000"+ + "\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000"+ + "\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000"+ + "\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%"+ + "\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001"+ "\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000"+ "\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u0000"+ "3\u0001\u0000\u0000\u0000\u00015\u0001\u0000\u0000\u0000\u00017\u0001"+ @@ -387,531 +385,528 @@ public EsqlBaseLexer(CharStream input) { "\u0097\u0001\u0000\u0000\u0000\u0002\u0099\u0001\u0000\u0000\u0000\u0002"+ "\u009b\u0001\u0000\u0000\u0000\u0002\u009d\u0001\u0000\u0000\u0000\u0002"+ "\u009f\u0001\u0000\u0000\u0000\u0002\u00a1\u0001\u0000\u0000\u0000\u0002"+ - "\u00a3\u0001\u0000\u0000\u0000\u0002\u00a5\u0001\u0000\u0000\u0000\u0002"+ - "\u00a7\u0001\u0000\u0000\u0000\u0002\u00a9\u0001\u0000\u0000\u0000\u0002"+ - "\u00ab\u0001\u0000\u0000\u0000\u0003\u00ad\u0001\u0000\u0000\u0000\u0003"+ - "\u00af\u0001\u0000\u0000\u0000\u0003\u00b1\u0001\u0000\u0000\u0000\u0003"+ - "\u00b3\u0001\u0000\u0000\u0000\u0003\u00b5\u0001\u0000\u0000\u0000\u0003"+ - "\u00b7\u0001\u0000\u0000\u0000\u0003\u00bb\u0001\u0000\u0000\u0000\u0003"+ - "\u00bd\u0001\u0000\u0000\u0000\u0003\u00bf\u0001\u0000\u0000\u0000\u0003"+ - "\u00c1\u0001\u0000\u0000\u0000\u0003\u00c3\u0001\u0000\u0000\u0000\u0004"+ - "\u00c5\u0001\u0000\u0000\u0000\u0004\u00c7\u0001\u0000\u0000\u0000\u0004"+ - "\u00c9\u0001\u0000\u0000\u0000\u0004\u00cd\u0001\u0000\u0000\u0000\u0004"+ - "\u00cf\u0001\u0000\u0000\u0000\u0004\u00d1\u0001\u0000\u0000\u0000\u0004"+ + "\u00a3\u0001\u0000\u0000\u0000\u0002\u00a7\u0001\u0000\u0000\u0000\u0002"+ + "\u00a9\u0001\u0000\u0000\u0000\u0002\u00ab\u0001\u0000\u0000\u0000\u0002"+ + "\u00ad\u0001\u0000\u0000\u0000\u0003\u00af\u0001\u0000\u0000\u0000\u0003"+ + "\u00b1\u0001\u0000\u0000\u0000\u0003\u00b3\u0001\u0000\u0000\u0000\u0003"+ + "\u00b5\u0001\u0000\u0000\u0000\u0003\u00b7\u0001\u0000\u0000\u0000\u0003"+ + "\u00b9\u0001\u0000\u0000\u0000\u0003\u00bd\u0001\u0000\u0000\u0000\u0003"+ + "\u00bf\u0001\u0000\u0000\u0000\u0003\u00c1\u0001\u0000\u0000\u0000\u0003"+ + "\u00c3\u0001\u0000\u0000\u0000\u0003\u00c5\u0001\u0000\u0000\u0000\u0004"+ + "\u00c7\u0001\u0000\u0000\u0000\u0004\u00c9\u0001\u0000\u0000\u0000\u0004"+ + "\u00cb\u0001\u0000\u0000\u0000\u0004\u00d1\u0001\u0000\u0000\u0000\u0004"+ "\u00d3\u0001\u0000\u0000\u0000\u0004\u00d5\u0001\u0000\u0000\u0000\u0004"+ "\u00d7\u0001\u0000\u0000\u0000\u0005\u00d9\u0001\u0000\u0000\u0000\u0005"+ "\u00db\u0001\u0000\u0000\u0000\u0005\u00dd\u0001\u0000\u0000\u0000\u0005"+ "\u00df\u0001\u0000\u0000\u0000\u0005\u00e1\u0001\u0000\u0000\u0000\u0005"+ "\u00e3\u0001\u0000\u0000\u0000\u0005\u00e5\u0001\u0000\u0000\u0000\u0005"+ - "\u00e7\u0001\u0000\u0000\u0000\u0005\u00e9\u0001\u0000\u0000\u0000\u0005"+ + "\u00e7\u0001\u0000\u0000\u0000\u0005\u00e9\u0001\u0000\u0000\u0000\u0006"+ "\u00eb\u0001\u0000\u0000\u0000\u0006\u00ed\u0001\u0000\u0000\u0000\u0006"+ "\u00ef\u0001\u0000\u0000\u0000\u0006\u00f1\u0001\u0000\u0000\u0000\u0006"+ - "\u00f3\u0001\u0000\u0000\u0000\u0006\u00f7\u0001\u0000\u0000\u0000\u0006"+ + "\u00f5\u0001\u0000\u0000\u0000\u0006\u00f7\u0001\u0000\u0000\u0000\u0006"+ "\u00f9\u0001\u0000\u0000\u0000\u0006\u00fb\u0001\u0000\u0000\u0000\u0006"+ - "\u00fd\u0001\u0000\u0000\u0000\u0006\u00ff\u0001\u0000\u0000\u0000\u0006"+ + "\u00fd\u0001\u0000\u0000\u0000\u0006\u00ff\u0001\u0000\u0000\u0000\u0007"+ "\u0101\u0001\u0000\u0000\u0000\u0007\u0103\u0001\u0000\u0000\u0000\u0007"+ "\u0105\u0001\u0000\u0000\u0000\u0007\u0107\u0001\u0000\u0000\u0000\u0007"+ "\u0109\u0001\u0000\u0000\u0000\u0007\u010b\u0001\u0000\u0000\u0000\u0007"+ "\u010d\u0001\u0000\u0000\u0000\u0007\u010f\u0001\u0000\u0000\u0000\u0007"+ - "\u0111\u0001\u0000\u0000\u0000\u0007\u0113\u0001\u0000\u0000\u0000\u0007"+ - "\u0115\u0001\u0000\u0000\u0000\b\u0117\u0001\u0000\u0000\u0000\b\u0119"+ - "\u0001\u0000\u0000\u0000\b\u011b\u0001\u0000\u0000\u0000\b\u011d\u0001"+ - "\u0000\u0000\u0000\b\u011f\u0001\u0000\u0000\u0000\b\u0121\u0001\u0000"+ - "\u0000\u0000\b\u0123\u0001\u0000\u0000\u0000\t\u0125\u0001\u0000\u0000"+ - "\u0000\t\u0127\u0001\u0000\u0000\u0000\t\u0129\u0001\u0000\u0000\u0000"+ - "\t\u012b\u0001\u0000\u0000\u0000\t\u012d\u0001\u0000\u0000\u0000\t\u012f"+ - "\u0001\u0000\u0000\u0000\n\u0131\u0001\u0000\u0000\u0000\n\u0133\u0001"+ - "\u0000\u0000\u0000\n\u0135\u0001\u0000\u0000\u0000\n\u0137\u0001\u0000"+ - "\u0000\u0000\n\u0139\u0001\u0000\u0000\u0000\n\u013b\u0001\u0000\u0000"+ - "\u0000\u000b\u013d\u0001\u0000\u0000\u0000\r\u0147\u0001\u0000\u0000\u0000"+ - "\u000f\u014e\u0001\u0000\u0000\u0000\u0011\u0157\u0001\u0000\u0000\u0000"+ - "\u0013\u015e\u0001\u0000\u0000\u0000\u0015\u0168\u0001\u0000\u0000\u0000"+ - "\u0017\u016f\u0001\u0000\u0000\u0000\u0019\u0176\u0001\u0000\u0000\u0000"+ - "\u001b\u0184\u0001\u0000\u0000\u0000\u001d\u018b\u0001\u0000\u0000\u0000"+ - "\u001f\u0193\u0001\u0000\u0000\u0000!\u019f\u0001\u0000\u0000\u0000#\u01a8"+ - "\u0001\u0000\u0000\u0000%\u01ae\u0001\u0000\u0000\u0000\'\u01b5\u0001"+ - "\u0000\u0000\u0000)\u01bc\u0001\u0000\u0000\u0000+\u01c4\u0001\u0000\u0000"+ - "\u0000-\u01cd\u0001\u0000\u0000\u0000/\u01d3\u0001\u0000\u0000\u00001"+ - "\u01e4\u0001\u0000\u0000\u00003\u01f4\u0001\u0000\u0000\u00005\u01fa\u0001"+ - "\u0000\u0000\u00007\u01ff\u0001\u0000\u0000\u00009\u0204\u0001\u0000\u0000"+ - "\u0000;\u0208\u0001\u0000\u0000\u0000=\u020c\u0001\u0000\u0000\u0000?"+ - "\u0210\u0001\u0000\u0000\u0000A\u0214\u0001\u0000\u0000\u0000C\u0216\u0001"+ - "\u0000\u0000\u0000E\u0218\u0001\u0000\u0000\u0000G\u021b\u0001\u0000\u0000"+ - "\u0000I\u021d\u0001\u0000\u0000\u0000K\u0226\u0001\u0000\u0000\u0000M"+ - "\u0228\u0001\u0000\u0000\u0000O\u022d\u0001\u0000\u0000\u0000Q\u022f\u0001"+ - "\u0000\u0000\u0000S\u0234\u0001\u0000\u0000\u0000U\u0253\u0001\u0000\u0000"+ - "\u0000W\u0256\u0001\u0000\u0000\u0000Y\u0284\u0001\u0000\u0000\u0000["+ - "\u0286\u0001\u0000\u0000\u0000]\u0289\u0001\u0000\u0000\u0000_\u028d\u0001"+ - "\u0000\u0000\u0000a\u0291\u0001\u0000\u0000\u0000c\u0293\u0001\u0000\u0000"+ - "\u0000e\u0295\u0001\u0000\u0000\u0000g\u029a\u0001\u0000\u0000\u0000i"+ - "\u029c\u0001\u0000\u0000\u0000k\u02a2\u0001\u0000\u0000\u0000m\u02a8\u0001"+ - "\u0000\u0000\u0000o\u02ad\u0001\u0000\u0000\u0000q\u02af\u0001\u0000\u0000"+ - "\u0000s\u02b2\u0001\u0000\u0000\u0000u\u02b5\u0001\u0000\u0000\u0000w"+ - "\u02ba\u0001\u0000\u0000\u0000y\u02be\u0001\u0000\u0000\u0000{\u02c3\u0001"+ - "\u0000\u0000\u0000}\u02c9\u0001\u0000\u0000\u0000\u007f\u02cc\u0001\u0000"+ - "\u0000\u0000\u0081\u02ce\u0001\u0000\u0000\u0000\u0083\u02d4\u0001\u0000"+ - "\u0000\u0000\u0085\u02d6\u0001\u0000\u0000\u0000\u0087\u02db\u0001\u0000"+ - "\u0000\u0000\u0089\u02de\u0001\u0000\u0000\u0000\u008b\u02e1\u0001\u0000"+ - "\u0000\u0000\u008d\u02e4\u0001\u0000\u0000\u0000\u008f\u02e6\u0001\u0000"+ - "\u0000\u0000\u0091\u02e9\u0001\u0000\u0000\u0000\u0093\u02eb\u0001\u0000"+ - "\u0000\u0000\u0095\u02ee\u0001\u0000\u0000\u0000\u0097\u02f0\u0001\u0000"+ - "\u0000\u0000\u0099\u02f2\u0001\u0000\u0000\u0000\u009b\u02f4\u0001\u0000"+ - "\u0000\u0000\u009d\u02f6\u0001\u0000\u0000\u0000\u009f\u02f8\u0001\u0000"+ - "\u0000\u0000\u00a1\u02fd\u0001\u0000\u0000\u0000\u00a3\u0312\u0001\u0000"+ - "\u0000\u0000\u00a5\u0314\u0001\u0000\u0000\u0000\u00a7\u031c\u0001\u0000"+ - "\u0000\u0000\u00a9\u0320\u0001\u0000\u0000\u0000\u00ab\u0324\u0001\u0000"+ - "\u0000\u0000\u00ad\u0328\u0001\u0000\u0000\u0000\u00af\u032d\u0001\u0000"+ - "\u0000\u0000\u00b1\u0331\u0001\u0000\u0000\u0000\u00b3\u0335\u0001\u0000"+ - "\u0000\u0000\u00b5\u0339\u0001\u0000\u0000\u0000\u00b7\u033d\u0001\u0000"+ - "\u0000\u0000\u00b9\u0349\u0001\u0000\u0000\u0000\u00bb\u034c\u0001\u0000"+ - "\u0000\u0000\u00bd\u0350\u0001\u0000\u0000\u0000\u00bf\u0354\u0001\u0000"+ - "\u0000\u0000\u00c1\u0358\u0001\u0000\u0000\u0000\u00c3\u035c\u0001\u0000"+ - "\u0000\u0000\u00c5\u0360\u0001\u0000\u0000\u0000\u00c7\u0365\u0001\u0000"+ - "\u0000\u0000\u00c9\u0369\u0001\u0000\u0000\u0000\u00cb\u0371\u0001\u0000"+ - "\u0000\u0000\u00cd\u0386\u0001\u0000\u0000\u0000\u00cf\u0388\u0001\u0000"+ - "\u0000\u0000\u00d1\u038c\u0001\u0000\u0000\u0000\u00d3\u0390\u0001\u0000"+ - "\u0000\u0000\u00d5\u0394\u0001\u0000\u0000\u0000\u00d7\u0398\u0001\u0000"+ - "\u0000\u0000\u00d9\u039c\u0001\u0000\u0000\u0000\u00db\u03a1\u0001\u0000"+ - "\u0000\u0000\u00dd\u03a5\u0001\u0000\u0000\u0000\u00df\u03a9\u0001\u0000"+ - "\u0000\u0000\u00e1\u03ad\u0001\u0000\u0000\u0000\u00e3\u03b0\u0001\u0000"+ - "\u0000\u0000\u00e5\u03b4\u0001\u0000\u0000\u0000\u00e7\u03b8\u0001\u0000"+ - "\u0000\u0000\u00e9\u03bc\u0001\u0000\u0000\u0000\u00eb\u03c0\u0001\u0000"+ - "\u0000\u0000\u00ed\u03c4\u0001\u0000\u0000\u0000\u00ef\u03c9\u0001\u0000"+ - "\u0000\u0000\u00f1\u03ce\u0001\u0000\u0000\u0000\u00f3\u03d3\u0001\u0000"+ - "\u0000\u0000\u00f5\u03da\u0001\u0000\u0000\u0000\u00f7\u03e3\u0001\u0000"+ - "\u0000\u0000\u00f9\u03ea\u0001\u0000\u0000\u0000\u00fb\u03ee\u0001\u0000"+ - "\u0000\u0000\u00fd\u03f2\u0001\u0000\u0000\u0000\u00ff\u03f6\u0001\u0000"+ - "\u0000\u0000\u0101\u03fa\u0001\u0000\u0000\u0000\u0103\u03fe\u0001\u0000"+ - "\u0000\u0000\u0105\u0404\u0001\u0000\u0000\u0000\u0107\u0408\u0001\u0000"+ - "\u0000\u0000\u0109\u040c\u0001\u0000\u0000\u0000\u010b\u0410\u0001\u0000"+ - "\u0000\u0000\u010d\u0414\u0001\u0000\u0000\u0000\u010f\u0418\u0001\u0000"+ - "\u0000\u0000\u0111\u041c\u0001\u0000\u0000\u0000\u0113\u0420\u0001\u0000"+ - "\u0000\u0000\u0115\u0424\u0001\u0000\u0000\u0000\u0117\u0428\u0001\u0000"+ - "\u0000\u0000\u0119\u042d\u0001\u0000\u0000\u0000\u011b\u0431\u0001\u0000"+ - "\u0000\u0000\u011d\u0435\u0001\u0000\u0000\u0000\u011f\u0439\u0001\u0000"+ - "\u0000\u0000\u0121\u043d\u0001\u0000\u0000\u0000\u0123\u0441\u0001\u0000"+ - "\u0000\u0000\u0125\u0445\u0001\u0000\u0000\u0000\u0127\u044a\u0001\u0000"+ - "\u0000\u0000\u0129\u044f\u0001\u0000\u0000\u0000\u012b\u0459\u0001\u0000"+ - "\u0000\u0000\u012d\u045d\u0001\u0000\u0000\u0000\u012f\u0461\u0001\u0000"+ - "\u0000\u0000\u0131\u0465\u0001\u0000\u0000\u0000\u0133\u046a\u0001\u0000"+ - "\u0000\u0000\u0135\u0471\u0001\u0000\u0000\u0000\u0137\u0475\u0001\u0000"+ - "\u0000\u0000\u0139\u0479\u0001\u0000\u0000\u0000\u013b\u047d\u0001\u0000"+ - "\u0000\u0000\u013d\u013e\u0005d\u0000\u0000\u013e\u013f\u0005i\u0000\u0000"+ - "\u013f\u0140\u0005s\u0000\u0000\u0140\u0141\u0005s\u0000\u0000\u0141\u0142"+ - "\u0005e\u0000\u0000\u0142\u0143\u0005c\u0000\u0000\u0143\u0144\u0005t"+ - "\u0000\u0000\u0144\u0145\u0001\u0000\u0000\u0000\u0145\u0146\u0006\u0000"+ - "\u0000\u0000\u0146\f\u0001\u0000\u0000\u0000\u0147\u0148\u0005d\u0000"+ - "\u0000\u0148\u0149\u0005r\u0000\u0000\u0149\u014a\u0005o\u0000\u0000\u014a"+ - "\u014b\u0005p\u0000\u0000\u014b\u014c\u0001\u0000\u0000\u0000\u014c\u014d"+ - "\u0006\u0001\u0001\u0000\u014d\u000e\u0001\u0000\u0000\u0000\u014e\u014f"+ - "\u0005e\u0000\u0000\u014f\u0150\u0005n\u0000\u0000\u0150\u0151\u0005r"+ - "\u0000\u0000\u0151\u0152\u0005i\u0000\u0000\u0152\u0153\u0005c\u0000\u0000"+ - "\u0153\u0154\u0005h\u0000\u0000\u0154\u0155\u0001\u0000\u0000\u0000\u0155"+ - "\u0156\u0006\u0002\u0002\u0000\u0156\u0010\u0001\u0000\u0000\u0000\u0157"+ - "\u0158\u0005e\u0000\u0000\u0158\u0159\u0005v\u0000\u0000\u0159\u015a\u0005"+ - "a\u0000\u0000\u015a\u015b\u0005l\u0000\u0000\u015b\u015c\u0001\u0000\u0000"+ - "\u0000\u015c\u015d\u0006\u0003\u0000\u0000\u015d\u0012\u0001\u0000\u0000"+ - "\u0000\u015e\u015f\u0005e\u0000\u0000\u015f\u0160\u0005x\u0000\u0000\u0160"+ - "\u0161\u0005p\u0000\u0000\u0161\u0162\u0005l\u0000\u0000\u0162\u0163\u0005"+ - "a\u0000\u0000\u0163\u0164\u0005i\u0000\u0000\u0164\u0165\u0005n\u0000"+ - "\u0000\u0165\u0166\u0001\u0000\u0000\u0000\u0166\u0167\u0006\u0004\u0003"+ - "\u0000\u0167\u0014\u0001\u0000\u0000\u0000\u0168\u0169\u0005f\u0000\u0000"+ - "\u0169\u016a\u0005r\u0000\u0000\u016a\u016b\u0005o\u0000\u0000\u016b\u016c"+ - "\u0005m\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d\u016e\u0006"+ - "\u0005\u0004\u0000\u016e\u0016\u0001\u0000\u0000\u0000\u016f\u0170\u0005"+ - "g\u0000\u0000\u0170\u0171\u0005r\u0000\u0000\u0171\u0172\u0005o\u0000"+ - "\u0000\u0172\u0173\u0005k\u0000\u0000\u0173\u0174\u0001\u0000\u0000\u0000"+ - "\u0174\u0175\u0006\u0006\u0000\u0000\u0175\u0018\u0001\u0000\u0000\u0000"+ - "\u0176\u0177\u0005i\u0000\u0000\u0177\u0178\u0005n\u0000\u0000\u0178\u0179"+ - "\u0005l\u0000\u0000\u0179\u017a\u0005i\u0000\u0000\u017a\u017b\u0005n"+ - "\u0000\u0000\u017b\u017c\u0005e\u0000\u0000\u017c\u017d\u0005s\u0000\u0000"+ - "\u017d\u017e\u0005t\u0000\u0000\u017e\u017f\u0005a\u0000\u0000\u017f\u0180"+ - "\u0005t\u0000\u0000\u0180\u0181\u0005s\u0000\u0000\u0181\u0182\u0001\u0000"+ - "\u0000\u0000\u0182\u0183\u0006\u0007\u0000\u0000\u0183\u001a\u0001\u0000"+ - "\u0000\u0000\u0184\u0185\u0005k\u0000\u0000\u0185\u0186\u0005e\u0000\u0000"+ - "\u0186\u0187\u0005e\u0000\u0000\u0187\u0188\u0005p\u0000\u0000\u0188\u0189"+ - "\u0001\u0000\u0000\u0000\u0189\u018a\u0006\b\u0001\u0000\u018a\u001c\u0001"+ - "\u0000\u0000\u0000\u018b\u018c\u0005l\u0000\u0000\u018c\u018d\u0005i\u0000"+ - "\u0000\u018d\u018e\u0005m\u0000\u0000\u018e\u018f\u0005i\u0000\u0000\u018f"+ - "\u0190\u0005t\u0000\u0000\u0190\u0191\u0001\u0000\u0000\u0000\u0191\u0192"+ - "\u0006\t\u0000\u0000\u0192\u001e\u0001\u0000\u0000\u0000\u0193\u0194\u0005"+ - "m\u0000\u0000\u0194\u0195\u0005v\u0000\u0000\u0195\u0196\u0005_\u0000"+ - "\u0000\u0196\u0197\u0005e\u0000\u0000\u0197\u0198\u0005x\u0000\u0000\u0198"+ - "\u0199\u0005p\u0000\u0000\u0199\u019a\u0005a\u0000\u0000\u019a\u019b\u0005"+ - "n\u0000\u0000\u019b\u019c\u0005d\u0000\u0000\u019c\u019d\u0001\u0000\u0000"+ - "\u0000\u019d\u019e\u0006\n\u0005\u0000\u019e \u0001\u0000\u0000\u0000"+ - "\u019f\u01a0\u0005r\u0000\u0000\u01a0\u01a1\u0005e\u0000\u0000\u01a1\u01a2"+ - "\u0005n\u0000\u0000\u01a2\u01a3\u0005a\u0000\u0000\u01a3\u01a4\u0005m"+ - "\u0000\u0000\u01a4\u01a5\u0005e\u0000\u0000\u01a5\u01a6\u0001\u0000\u0000"+ - "\u0000\u01a6\u01a7\u0006\u000b\u0006\u0000\u01a7\"\u0001\u0000\u0000\u0000"+ - "\u01a8\u01a9\u0005r\u0000\u0000\u01a9\u01aa\u0005o\u0000\u0000\u01aa\u01ab"+ - "\u0005w\u0000\u0000\u01ab\u01ac\u0001\u0000\u0000\u0000\u01ac\u01ad\u0006"+ - "\f\u0000\u0000\u01ad$\u0001\u0000\u0000\u0000\u01ae\u01af\u0005s\u0000"+ - "\u0000\u01af\u01b0\u0005h\u0000\u0000\u01b0\u01b1\u0005o\u0000\u0000\u01b1"+ - "\u01b2\u0005w\u0000\u0000\u01b2\u01b3\u0001\u0000\u0000\u0000\u01b3\u01b4"+ - "\u0006\r\u0007\u0000\u01b4&\u0001\u0000\u0000\u0000\u01b5\u01b6\u0005"+ - "s\u0000\u0000\u01b6\u01b7\u0005o\u0000\u0000\u01b7\u01b8\u0005r\u0000"+ - "\u0000\u01b8\u01b9\u0005t\u0000\u0000\u01b9\u01ba\u0001\u0000\u0000\u0000"+ - "\u01ba\u01bb\u0006\u000e\u0000\u0000\u01bb(\u0001\u0000\u0000\u0000\u01bc"+ - "\u01bd\u0005s\u0000\u0000\u01bd\u01be\u0005t\u0000\u0000\u01be\u01bf\u0005"+ - "a\u0000\u0000\u01bf\u01c0\u0005t\u0000\u0000\u01c0\u01c1\u0005s\u0000"+ - "\u0000\u01c1\u01c2\u0001\u0000\u0000\u0000\u01c2\u01c3\u0006\u000f\u0000"+ - "\u0000\u01c3*\u0001\u0000\u0000\u0000\u01c4\u01c5\u0005w\u0000\u0000\u01c5"+ - "\u01c6\u0005h\u0000\u0000\u01c6\u01c7\u0005e\u0000\u0000\u01c7\u01c8\u0005"+ - "r\u0000\u0000\u01c8\u01c9\u0005e\u0000\u0000\u01c9\u01ca\u0001\u0000\u0000"+ - "\u0000\u01ca\u01cb\u0006\u0010\u0000\u0000\u01cb,\u0001\u0000\u0000\u0000"+ - "\u01cc\u01ce\b\u0000\u0000\u0000\u01cd\u01cc\u0001\u0000\u0000\u0000\u01ce"+ - "\u01cf\u0001\u0000\u0000\u0000\u01cf\u01cd\u0001\u0000\u0000\u0000\u01cf"+ - "\u01d0\u0001\u0000\u0000\u0000\u01d0\u01d1\u0001\u0000\u0000\u0000\u01d1"+ - "\u01d2\u0006\u0011\u0000\u0000\u01d2.\u0001\u0000\u0000\u0000\u01d3\u01d4"+ - "\u0005/\u0000\u0000\u01d4\u01d5\u0005/\u0000\u0000\u01d5\u01d9\u0001\u0000"+ - "\u0000\u0000\u01d6\u01d8\b\u0001\u0000\u0000\u01d7\u01d6\u0001\u0000\u0000"+ - "\u0000\u01d8\u01db\u0001\u0000\u0000\u0000\u01d9\u01d7\u0001\u0000\u0000"+ - "\u0000\u01d9\u01da\u0001\u0000\u0000\u0000\u01da\u01dd\u0001\u0000\u0000"+ - "\u0000\u01db\u01d9\u0001\u0000\u0000\u0000\u01dc\u01de\u0005\r\u0000\u0000"+ - "\u01dd\u01dc\u0001\u0000\u0000\u0000\u01dd\u01de\u0001\u0000\u0000\u0000"+ - "\u01de\u01e0\u0001\u0000\u0000\u0000\u01df\u01e1\u0005\n\u0000\u0000\u01e0"+ - "\u01df\u0001\u0000\u0000\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e1"+ - "\u01e2\u0001\u0000\u0000\u0000\u01e2\u01e3\u0006\u0012\b\u0000\u01e30"+ - "\u0001\u0000\u0000\u0000\u01e4\u01e5\u0005/\u0000\u0000\u01e5\u01e6\u0005"+ - "*\u0000\u0000\u01e6\u01eb\u0001\u0000\u0000\u0000\u01e7\u01ea\u00031\u0013"+ - "\u0000\u01e8\u01ea\t\u0000\u0000\u0000\u01e9\u01e7\u0001\u0000\u0000\u0000"+ - "\u01e9\u01e8\u0001\u0000\u0000\u0000\u01ea\u01ed\u0001\u0000\u0000\u0000"+ - "\u01eb\u01ec\u0001\u0000\u0000\u0000\u01eb\u01e9\u0001\u0000\u0000\u0000"+ - "\u01ec\u01ee\u0001\u0000\u0000\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+ - "\u01ee\u01ef\u0005*\u0000\u0000\u01ef\u01f0\u0005/\u0000\u0000\u01f0\u01f1"+ - "\u0001\u0000\u0000\u0000\u01f1\u01f2\u0006\u0013\b\u0000\u01f22\u0001"+ - "\u0000\u0000\u0000\u01f3\u01f5\u0007\u0002\u0000\u0000\u01f4\u01f3\u0001"+ - "\u0000\u0000\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f4\u0001"+ - "\u0000\u0000\u0000\u01f6\u01f7\u0001\u0000\u0000\u0000\u01f7\u01f8\u0001"+ - "\u0000\u0000\u0000\u01f8\u01f9\u0006\u0014\b\u0000\u01f94\u0001\u0000"+ - "\u0000\u0000\u01fa\u01fb\u0003\u009fJ\u0000\u01fb\u01fc\u0001\u0000\u0000"+ - "\u0000\u01fc\u01fd\u0006\u0015\t\u0000\u01fd\u01fe\u0006\u0015\n\u0000"+ - "\u01fe6\u0001\u0000\u0000\u0000\u01ff\u0200\u0003?\u001a\u0000\u0200\u0201"+ - "\u0001\u0000\u0000\u0000\u0201\u0202\u0006\u0016\u000b\u0000\u0202\u0203"+ - "\u0006\u0016\f\u0000\u02038\u0001\u0000\u0000\u0000\u0204\u0205\u0003"+ - "3\u0014\u0000\u0205\u0206\u0001\u0000\u0000\u0000\u0206\u0207\u0006\u0017"+ - "\b\u0000\u0207:\u0001\u0000\u0000\u0000\u0208\u0209\u0003/\u0012\u0000"+ - "\u0209\u020a\u0001\u0000\u0000\u0000\u020a\u020b\u0006\u0018\b\u0000\u020b"+ - "<\u0001\u0000\u0000\u0000\u020c\u020d\u00031\u0013\u0000\u020d\u020e\u0001"+ - "\u0000\u0000\u0000\u020e\u020f\u0006\u0019\b\u0000\u020f>\u0001\u0000"+ - "\u0000\u0000\u0210\u0211\u0005|\u0000\u0000\u0211\u0212\u0001\u0000\u0000"+ - "\u0000\u0212\u0213\u0006\u001a\f\u0000\u0213@\u0001\u0000\u0000\u0000"+ - "\u0214\u0215\u0007\u0003\u0000\u0000\u0215B\u0001\u0000\u0000\u0000\u0216"+ - "\u0217\u0007\u0004\u0000\u0000\u0217D\u0001\u0000\u0000\u0000\u0218\u0219"+ - "\u0005\\\u0000\u0000\u0219\u021a\u0007\u0005\u0000\u0000\u021aF\u0001"+ - "\u0000\u0000\u0000\u021b\u021c\b\u0006\u0000\u0000\u021cH\u0001\u0000"+ - "\u0000\u0000\u021d\u021f\u0007\u0007\u0000\u0000\u021e\u0220\u0007\b\u0000"+ - "\u0000\u021f\u021e\u0001\u0000\u0000\u0000\u021f\u0220\u0001\u0000\u0000"+ - "\u0000\u0220\u0222\u0001\u0000\u0000\u0000\u0221\u0223\u0003A\u001b\u0000"+ - "\u0222\u0221\u0001\u0000\u0000\u0000\u0223\u0224\u0001\u0000\u0000\u0000"+ - "\u0224\u0222\u0001\u0000\u0000\u0000\u0224\u0225\u0001\u0000\u0000\u0000"+ - "\u0225J\u0001\u0000\u0000\u0000\u0226\u0227\u0005@\u0000\u0000\u0227L"+ - "\u0001\u0000\u0000\u0000\u0228\u0229\u0005`\u0000\u0000\u0229N\u0001\u0000"+ - "\u0000\u0000\u022a\u022e\b\t\u0000\u0000\u022b\u022c\u0005`\u0000\u0000"+ - "\u022c\u022e\u0005`\u0000\u0000\u022d\u022a\u0001\u0000\u0000\u0000\u022d"+ - "\u022b\u0001\u0000\u0000\u0000\u022eP\u0001\u0000\u0000\u0000\u022f\u0230"+ - "\u0005_\u0000\u0000\u0230R\u0001\u0000\u0000\u0000\u0231\u0235\u0003C"+ - "\u001c\u0000\u0232\u0235\u0003A\u001b\u0000\u0233\u0235\u0003Q#\u0000"+ - "\u0234\u0231\u0001\u0000\u0000\u0000\u0234\u0232\u0001\u0000\u0000\u0000"+ - "\u0234\u0233\u0001\u0000\u0000\u0000\u0235T\u0001\u0000\u0000\u0000\u0236"+ - "\u023b\u0005\"\u0000\u0000\u0237\u023a\u0003E\u001d\u0000\u0238\u023a"+ - "\u0003G\u001e\u0000\u0239\u0237\u0001\u0000\u0000\u0000\u0239\u0238\u0001"+ - "\u0000\u0000\u0000\u023a\u023d\u0001\u0000\u0000\u0000\u023b\u0239\u0001"+ - "\u0000\u0000\u0000\u023b\u023c\u0001\u0000\u0000\u0000\u023c\u023e\u0001"+ - "\u0000\u0000\u0000\u023d\u023b\u0001\u0000\u0000\u0000\u023e\u0254\u0005"+ - "\"\u0000\u0000\u023f\u0240\u0005\"\u0000\u0000\u0240\u0241\u0005\"\u0000"+ - "\u0000\u0241\u0242\u0005\"\u0000\u0000\u0242\u0246\u0001\u0000\u0000\u0000"+ - "\u0243\u0245\b\u0001\u0000\u0000\u0244\u0243\u0001\u0000\u0000\u0000\u0245"+ - "\u0248\u0001\u0000\u0000\u0000\u0246\u0247\u0001\u0000\u0000\u0000\u0246"+ - "\u0244\u0001\u0000\u0000\u0000\u0247\u0249\u0001\u0000\u0000\u0000\u0248"+ - "\u0246\u0001\u0000\u0000\u0000\u0249\u024a\u0005\"\u0000\u0000\u024a\u024b"+ - "\u0005\"\u0000\u0000\u024b\u024c\u0005\"\u0000\u0000\u024c\u024e\u0001"+ - "\u0000\u0000\u0000\u024d\u024f\u0005\"\u0000\u0000\u024e\u024d\u0001\u0000"+ - "\u0000\u0000\u024e\u024f\u0001\u0000\u0000\u0000\u024f\u0251\u0001\u0000"+ - "\u0000\u0000\u0250\u0252\u0005\"\u0000\u0000\u0251\u0250\u0001\u0000\u0000"+ - "\u0000\u0251\u0252\u0001\u0000\u0000\u0000\u0252\u0254\u0001\u0000\u0000"+ - "\u0000\u0253\u0236\u0001\u0000\u0000\u0000\u0253\u023f\u0001\u0000\u0000"+ - "\u0000\u0254V\u0001\u0000\u0000\u0000\u0255\u0257\u0003A\u001b\u0000\u0256"+ - "\u0255\u0001\u0000\u0000\u0000\u0257\u0258\u0001\u0000\u0000\u0000\u0258"+ - "\u0256\u0001\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000\u0259"+ - "X\u0001\u0000\u0000\u0000\u025a\u025c\u0003A\u001b\u0000\u025b\u025a\u0001"+ - "\u0000\u0000\u0000\u025c\u025d\u0001\u0000\u0000\u0000\u025d\u025b\u0001"+ - "\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000\u025e\u025f\u0001"+ - "\u0000\u0000\u0000\u025f\u0263\u0003g.\u0000\u0260\u0262\u0003A\u001b"+ - "\u0000\u0261\u0260\u0001\u0000\u0000\u0000\u0262\u0265\u0001\u0000\u0000"+ - "\u0000\u0263\u0261\u0001\u0000\u0000\u0000\u0263\u0264\u0001\u0000\u0000"+ - "\u0000\u0264\u0285\u0001\u0000\u0000\u0000\u0265\u0263\u0001\u0000\u0000"+ - "\u0000\u0266\u0268\u0003g.\u0000\u0267\u0269\u0003A\u001b\u0000\u0268"+ - "\u0267\u0001\u0000\u0000\u0000\u0269\u026a\u0001\u0000\u0000\u0000\u026a"+ - "\u0268\u0001\u0000\u0000\u0000\u026a\u026b\u0001\u0000\u0000\u0000\u026b"+ - "\u0285\u0001\u0000\u0000\u0000\u026c\u026e\u0003A\u001b\u0000\u026d\u026c"+ - "\u0001\u0000\u0000\u0000\u026e\u026f\u0001\u0000\u0000\u0000\u026f\u026d"+ - "\u0001\u0000\u0000\u0000\u026f\u0270\u0001\u0000\u0000\u0000\u0270\u0278"+ - "\u0001\u0000\u0000\u0000\u0271\u0275\u0003g.\u0000\u0272\u0274\u0003A"+ - "\u001b\u0000\u0273\u0272\u0001\u0000\u0000\u0000\u0274\u0277\u0001\u0000"+ - "\u0000\u0000\u0275\u0273\u0001\u0000\u0000\u0000\u0275\u0276\u0001\u0000"+ - "\u0000\u0000\u0276\u0279\u0001\u0000\u0000\u0000\u0277\u0275\u0001\u0000"+ - "\u0000\u0000\u0278\u0271\u0001\u0000\u0000\u0000\u0278\u0279\u0001\u0000"+ - "\u0000\u0000\u0279\u027a\u0001\u0000\u0000\u0000\u027a\u027b\u0003I\u001f"+ - "\u0000\u027b\u0285\u0001\u0000\u0000\u0000\u027c\u027e\u0003g.\u0000\u027d"+ - "\u027f\u0003A\u001b\u0000\u027e\u027d\u0001\u0000\u0000\u0000\u027f\u0280"+ - "\u0001\u0000\u0000\u0000\u0280\u027e\u0001\u0000\u0000\u0000\u0280\u0281"+ - "\u0001\u0000\u0000\u0000\u0281\u0282\u0001\u0000\u0000\u0000\u0282\u0283"+ - "\u0003I\u001f\u0000\u0283\u0285\u0001\u0000\u0000\u0000\u0284\u025b\u0001"+ - "\u0000\u0000\u0000\u0284\u0266\u0001\u0000\u0000\u0000\u0284\u026d\u0001"+ - "\u0000\u0000\u0000\u0284\u027c\u0001\u0000\u0000\u0000\u0285Z\u0001\u0000"+ - "\u0000\u0000\u0286\u0287\u0005b\u0000\u0000\u0287\u0288\u0005y\u0000\u0000"+ - "\u0288\\\u0001\u0000\u0000\u0000\u0289\u028a\u0005a\u0000\u0000\u028a"+ - "\u028b\u0005n\u0000\u0000\u028b\u028c\u0005d\u0000\u0000\u028c^\u0001"+ - "\u0000\u0000\u0000\u028d\u028e\u0005a\u0000\u0000\u028e\u028f\u0005s\u0000"+ - "\u0000\u028f\u0290\u0005c\u0000\u0000\u0290`\u0001\u0000\u0000\u0000\u0291"+ - "\u0292\u0005=\u0000\u0000\u0292b\u0001\u0000\u0000\u0000\u0293\u0294\u0005"+ - ",\u0000\u0000\u0294d\u0001\u0000\u0000\u0000\u0295\u0296\u0005d\u0000"+ - "\u0000\u0296\u0297\u0005e\u0000\u0000\u0297\u0298\u0005s\u0000\u0000\u0298"+ - "\u0299\u0005c\u0000\u0000\u0299f\u0001\u0000\u0000\u0000\u029a\u029b\u0005"+ - ".\u0000\u0000\u029bh\u0001\u0000\u0000\u0000\u029c\u029d\u0005f\u0000"+ - "\u0000\u029d\u029e\u0005a\u0000\u0000\u029e\u029f\u0005l\u0000\u0000\u029f"+ - "\u02a0\u0005s\u0000\u0000\u02a0\u02a1\u0005e\u0000\u0000\u02a1j\u0001"+ - "\u0000\u0000\u0000\u02a2\u02a3\u0005f\u0000\u0000\u02a3\u02a4\u0005i\u0000"+ - "\u0000\u02a4\u02a5\u0005r\u0000\u0000\u02a5\u02a6\u0005s\u0000\u0000\u02a6"+ - "\u02a7\u0005t\u0000\u0000\u02a7l\u0001\u0000\u0000\u0000\u02a8\u02a9\u0005"+ - "l\u0000\u0000\u02a9\u02aa\u0005a\u0000\u0000\u02aa\u02ab\u0005s\u0000"+ - "\u0000\u02ab\u02ac\u0005t\u0000\u0000\u02acn\u0001\u0000\u0000\u0000\u02ad"+ - "\u02ae\u0005(\u0000\u0000\u02aep\u0001\u0000\u0000\u0000\u02af\u02b0\u0005"+ - "i\u0000\u0000\u02b0\u02b1\u0005n\u0000\u0000\u02b1r\u0001\u0000\u0000"+ - "\u0000\u02b2\u02b3\u0005i\u0000\u0000\u02b3\u02b4\u0005s\u0000\u0000\u02b4"+ - "t\u0001\u0000\u0000\u0000\u02b5\u02b6\u0005l\u0000\u0000\u02b6\u02b7\u0005"+ - "i\u0000\u0000\u02b7\u02b8\u0005k\u0000\u0000\u02b8\u02b9\u0005e\u0000"+ - "\u0000\u02b9v\u0001\u0000\u0000\u0000\u02ba\u02bb\u0005n\u0000\u0000\u02bb"+ - "\u02bc\u0005o\u0000\u0000\u02bc\u02bd\u0005t\u0000\u0000\u02bdx\u0001"+ - "\u0000\u0000\u0000\u02be\u02bf\u0005n\u0000\u0000\u02bf\u02c0\u0005u\u0000"+ - "\u0000\u02c0\u02c1\u0005l\u0000\u0000\u02c1\u02c2\u0005l\u0000\u0000\u02c2"+ - "z\u0001\u0000\u0000\u0000\u02c3\u02c4\u0005n\u0000\u0000\u02c4\u02c5\u0005"+ - "u\u0000\u0000\u02c5\u02c6\u0005l\u0000\u0000\u02c6\u02c7\u0005l\u0000"+ - "\u0000\u02c7\u02c8\u0005s\u0000\u0000\u02c8|\u0001\u0000\u0000\u0000\u02c9"+ - "\u02ca\u0005o\u0000\u0000\u02ca\u02cb\u0005r\u0000\u0000\u02cb~\u0001"+ - "\u0000\u0000\u0000\u02cc\u02cd\u0005?\u0000\u0000\u02cd\u0080\u0001\u0000"+ - "\u0000\u0000\u02ce\u02cf\u0005r\u0000\u0000\u02cf\u02d0\u0005l\u0000\u0000"+ - "\u02d0\u02d1\u0005i\u0000\u0000\u02d1\u02d2\u0005k\u0000\u0000\u02d2\u02d3"+ - "\u0005e\u0000\u0000\u02d3\u0082\u0001\u0000\u0000\u0000\u02d4\u02d5\u0005"+ - ")\u0000\u0000\u02d5\u0084\u0001\u0000\u0000\u0000\u02d6\u02d7\u0005t\u0000"+ - "\u0000\u02d7\u02d8\u0005r\u0000\u0000\u02d8\u02d9\u0005u\u0000\u0000\u02d9"+ - "\u02da\u0005e\u0000\u0000\u02da\u0086\u0001\u0000\u0000\u0000\u02db\u02dc"+ - "\u0005=\u0000\u0000\u02dc\u02dd\u0005=\u0000\u0000\u02dd\u0088\u0001\u0000"+ - "\u0000\u0000\u02de\u02df\u0005=\u0000\u0000\u02df\u02e0\u0005~\u0000\u0000"+ - "\u02e0\u008a\u0001\u0000\u0000\u0000\u02e1\u02e2\u0005!\u0000\u0000\u02e2"+ - "\u02e3\u0005=\u0000\u0000\u02e3\u008c\u0001\u0000\u0000\u0000\u02e4\u02e5"+ - "\u0005<\u0000\u0000\u02e5\u008e\u0001\u0000\u0000\u0000\u02e6\u02e7\u0005"+ - "<\u0000\u0000\u02e7\u02e8\u0005=\u0000\u0000\u02e8\u0090\u0001\u0000\u0000"+ - "\u0000\u02e9\u02ea\u0005>\u0000\u0000\u02ea\u0092\u0001\u0000\u0000\u0000"+ - "\u02eb\u02ec\u0005>\u0000\u0000\u02ec\u02ed\u0005=\u0000\u0000\u02ed\u0094"+ - "\u0001\u0000\u0000\u0000\u02ee\u02ef\u0005+\u0000\u0000\u02ef\u0096\u0001"+ - "\u0000\u0000\u0000\u02f0\u02f1\u0005-\u0000\u0000\u02f1\u0098\u0001\u0000"+ - "\u0000\u0000\u02f2\u02f3\u0005*\u0000\u0000\u02f3\u009a\u0001\u0000\u0000"+ - "\u0000\u02f4\u02f5\u0005/\u0000\u0000\u02f5\u009c\u0001\u0000\u0000\u0000"+ - "\u02f6\u02f7\u0005%\u0000\u0000\u02f7\u009e\u0001\u0000\u0000\u0000\u02f8"+ - "\u02f9\u0005[\u0000\u0000\u02f9\u02fa\u0001\u0000\u0000\u0000\u02fa\u02fb"+ - "\u0006J\u0000\u0000\u02fb\u02fc\u0006J\u0000\u0000\u02fc\u00a0\u0001\u0000"+ - "\u0000\u0000\u02fd\u02fe\u0005]\u0000\u0000\u02fe\u02ff\u0001\u0000\u0000"+ - "\u0000\u02ff\u0300\u0006K\f\u0000\u0300\u0301\u0006K\f\u0000\u0301\u00a2"+ - "\u0001\u0000\u0000\u0000\u0302\u0306\u0003C\u001c\u0000\u0303\u0305\u0003"+ - "S$\u0000\u0304\u0303\u0001\u0000\u0000\u0000\u0305\u0308\u0001\u0000\u0000"+ - "\u0000\u0306\u0304\u0001\u0000\u0000\u0000\u0306\u0307\u0001\u0000\u0000"+ - "\u0000\u0307\u0313\u0001\u0000\u0000\u0000\u0308\u0306\u0001\u0000\u0000"+ - "\u0000\u0309\u030c\u0003Q#\u0000\u030a\u030c\u0003K \u0000\u030b\u0309"+ - "\u0001\u0000\u0000\u0000\u030b\u030a\u0001\u0000\u0000\u0000\u030c\u030e"+ - "\u0001\u0000\u0000\u0000\u030d\u030f\u0003S$\u0000\u030e\u030d\u0001\u0000"+ - "\u0000\u0000\u030f\u0310\u0001\u0000\u0000\u0000\u0310\u030e\u0001\u0000"+ - "\u0000\u0000\u0310\u0311\u0001\u0000\u0000\u0000\u0311\u0313\u0001\u0000"+ - "\u0000\u0000\u0312\u0302\u0001\u0000\u0000\u0000\u0312\u030b\u0001\u0000"+ - "\u0000\u0000\u0313\u00a4\u0001\u0000\u0000\u0000\u0314\u0316\u0003M!\u0000"+ - "\u0315\u0317\u0003O\"\u0000\u0316\u0315\u0001\u0000\u0000\u0000\u0317"+ - "\u0318\u0001\u0000\u0000\u0000\u0318\u0316\u0001\u0000\u0000\u0000\u0318"+ - "\u0319\u0001\u0000\u0000\u0000\u0319\u031a\u0001\u0000\u0000\u0000\u031a"+ - "\u031b\u0003M!\u0000\u031b\u00a6\u0001\u0000\u0000\u0000\u031c\u031d\u0003"+ - "/\u0012\u0000\u031d\u031e\u0001\u0000\u0000\u0000\u031e\u031f\u0006N\b"+ - "\u0000\u031f\u00a8\u0001\u0000\u0000\u0000\u0320\u0321\u00031\u0013\u0000"+ - "\u0321\u0322\u0001\u0000\u0000\u0000\u0322\u0323\u0006O\b\u0000\u0323"+ - "\u00aa\u0001\u0000\u0000\u0000\u0324\u0325\u00033\u0014\u0000\u0325\u0326"+ - "\u0001\u0000\u0000\u0000\u0326\u0327\u0006P\b\u0000\u0327\u00ac\u0001"+ - "\u0000\u0000\u0000\u0328\u0329\u0003?\u001a\u0000\u0329\u032a\u0001\u0000"+ - "\u0000\u0000\u032a\u032b\u0006Q\u000b\u0000\u032b\u032c\u0006Q\f\u0000"+ - "\u032c\u00ae\u0001\u0000\u0000\u0000\u032d\u032e\u0003\u009fJ\u0000\u032e"+ - "\u032f\u0001\u0000\u0000\u0000\u032f\u0330\u0006R\t\u0000\u0330\u00b0"+ - "\u0001\u0000\u0000\u0000\u0331\u0332\u0003\u00a1K\u0000\u0332\u0333\u0001"+ - "\u0000\u0000\u0000\u0333\u0334\u0006S\r\u0000\u0334\u00b2\u0001\u0000"+ - "\u0000\u0000\u0335\u0336\u0003c,\u0000\u0336\u0337\u0001\u0000\u0000\u0000"+ - "\u0337\u0338\u0006T\u000e\u0000\u0338\u00b4\u0001\u0000\u0000\u0000\u0339"+ - "\u033a\u0003a+\u0000\u033a\u033b\u0001\u0000\u0000\u0000\u033b\u033c\u0006"+ - "U\u000f\u0000\u033c\u00b6\u0001\u0000\u0000\u0000\u033d\u033e\u0005m\u0000"+ + "\u0111\u0001\u0000\u0000\u0000\u0007\u0113\u0001\u0000\u0000\u0000\b\u0115"+ + "\u0001\u0000\u0000\u0000\b\u0117\u0001\u0000\u0000\u0000\b\u0119\u0001"+ + "\u0000\u0000\u0000\b\u011b\u0001\u0000\u0000\u0000\b\u011d\u0001\u0000"+ + "\u0000\u0000\b\u011f\u0001\u0000\u0000\u0000\b\u0121\u0001\u0000\u0000"+ + "\u0000\t\u0123\u0001\u0000\u0000\u0000\t\u0125\u0001\u0000\u0000\u0000"+ + "\t\u0127\u0001\u0000\u0000\u0000\t\u0129\u0001\u0000\u0000\u0000\t\u012b"+ + "\u0001\u0000\u0000\u0000\t\u012d\u0001\u0000\u0000\u0000\n\u012f\u0001"+ + "\u0000\u0000\u0000\n\u0131\u0001\u0000\u0000\u0000\n\u0133\u0001\u0000"+ + "\u0000\u0000\n\u0135\u0001\u0000\u0000\u0000\n\u0137\u0001\u0000\u0000"+ + "\u0000\n\u0139\u0001\u0000\u0000\u0000\u000b\u013b\u0001\u0000\u0000\u0000"+ + "\r\u0145\u0001\u0000\u0000\u0000\u000f\u014c\u0001\u0000\u0000\u0000\u0011"+ + "\u0155\u0001\u0000\u0000\u0000\u0013\u015c\u0001\u0000\u0000\u0000\u0015"+ + "\u0166\u0001\u0000\u0000\u0000\u0017\u016d\u0001\u0000\u0000\u0000\u0019"+ + "\u0174\u0001\u0000\u0000\u0000\u001b\u0182\u0001\u0000\u0000\u0000\u001d"+ + "\u0189\u0001\u0000\u0000\u0000\u001f\u0191\u0001\u0000\u0000\u0000!\u019d"+ + "\u0001\u0000\u0000\u0000#\u01a6\u0001\u0000\u0000\u0000%\u01ac\u0001\u0000"+ + "\u0000\u0000\'\u01b3\u0001\u0000\u0000\u0000)\u01ba\u0001\u0000\u0000"+ + "\u0000+\u01c2\u0001\u0000\u0000\u0000-\u01cb\u0001\u0000\u0000\u0000/"+ + "\u01d1\u0001\u0000\u0000\u00001\u01e2\u0001\u0000\u0000\u00003\u01f2\u0001"+ + "\u0000\u0000\u00005\u01f8\u0001\u0000\u0000\u00007\u01fd\u0001\u0000\u0000"+ + "\u00009\u0202\u0001\u0000\u0000\u0000;\u0206\u0001\u0000\u0000\u0000="+ + "\u020a\u0001\u0000\u0000\u0000?\u020e\u0001\u0000\u0000\u0000A\u0212\u0001"+ + "\u0000\u0000\u0000C\u0214\u0001\u0000\u0000\u0000E\u0216\u0001\u0000\u0000"+ + "\u0000G\u0219\u0001\u0000\u0000\u0000I\u021b\u0001\u0000\u0000\u0000K"+ + "\u0224\u0001\u0000\u0000\u0000M\u0226\u0001\u0000\u0000\u0000O\u022b\u0001"+ + "\u0000\u0000\u0000Q\u022d\u0001\u0000\u0000\u0000S\u0232\u0001\u0000\u0000"+ + "\u0000U\u0251\u0001\u0000\u0000\u0000W\u0254\u0001\u0000\u0000\u0000Y"+ + "\u0282\u0001\u0000\u0000\u0000[\u0284\u0001\u0000\u0000\u0000]\u0287\u0001"+ + "\u0000\u0000\u0000_\u028b\u0001\u0000\u0000\u0000a\u028f\u0001\u0000\u0000"+ + "\u0000c\u0291\u0001\u0000\u0000\u0000e\u0293\u0001\u0000\u0000\u0000g"+ + "\u0298\u0001\u0000\u0000\u0000i\u029a\u0001\u0000\u0000\u0000k\u02a0\u0001"+ + "\u0000\u0000\u0000m\u02a6\u0001\u0000\u0000\u0000o\u02ab\u0001\u0000\u0000"+ + "\u0000q\u02ad\u0001\u0000\u0000\u0000s\u02b0\u0001\u0000\u0000\u0000u"+ + "\u02b3\u0001\u0000\u0000\u0000w\u02b8\u0001\u0000\u0000\u0000y\u02bc\u0001"+ + "\u0000\u0000\u0000{\u02c1\u0001\u0000\u0000\u0000}\u02c7\u0001\u0000\u0000"+ + "\u0000\u007f\u02ca\u0001\u0000\u0000\u0000\u0081\u02cc\u0001\u0000\u0000"+ + "\u0000\u0083\u02d2\u0001\u0000\u0000\u0000\u0085\u02d4\u0001\u0000\u0000"+ + "\u0000\u0087\u02d9\u0001\u0000\u0000\u0000\u0089\u02dc\u0001\u0000\u0000"+ + "\u0000\u008b\u02df\u0001\u0000\u0000\u0000\u008d\u02e2\u0001\u0000\u0000"+ + "\u0000\u008f\u02e4\u0001\u0000\u0000\u0000\u0091\u02e7\u0001\u0000\u0000"+ + "\u0000\u0093\u02e9\u0001\u0000\u0000\u0000\u0095\u02ec\u0001\u0000\u0000"+ + "\u0000\u0097\u02ee\u0001\u0000\u0000\u0000\u0099\u02f0\u0001\u0000\u0000"+ + "\u0000\u009b\u02f2\u0001\u0000\u0000\u0000\u009d\u02f4\u0001\u0000\u0000"+ + "\u0000\u009f\u02f6\u0001\u0000\u0000\u0000\u00a1\u02fb\u0001\u0000\u0000"+ + "\u0000\u00a3\u0310\u0001\u0000\u0000\u0000\u00a5\u0312\u0001\u0000\u0000"+ + "\u0000\u00a7\u031a\u0001\u0000\u0000\u0000\u00a9\u031c\u0001\u0000\u0000"+ + "\u0000\u00ab\u0320\u0001\u0000\u0000\u0000\u00ad\u0324\u0001\u0000\u0000"+ + "\u0000\u00af\u0328\u0001\u0000\u0000\u0000\u00b1\u032d\u0001\u0000\u0000"+ + "\u0000\u00b3\u0331\u0001\u0000\u0000\u0000\u00b5\u0335\u0001\u0000\u0000"+ + "\u0000\u00b7\u0339\u0001\u0000\u0000\u0000\u00b9\u033d\u0001\u0000\u0000"+ + "\u0000\u00bb\u0349\u0001\u0000\u0000\u0000\u00bd\u034c\u0001\u0000\u0000"+ + "\u0000\u00bf\u0350\u0001\u0000\u0000\u0000\u00c1\u0354\u0001\u0000\u0000"+ + "\u0000\u00c3\u0358\u0001\u0000\u0000\u0000\u00c5\u035c\u0001\u0000\u0000"+ + "\u0000\u00c7\u0360\u0001\u0000\u0000\u0000\u00c9\u0365\u0001\u0000\u0000"+ + "\u0000\u00cb\u0369\u0001\u0000\u0000\u0000\u00cd\u0371\u0001\u0000\u0000"+ + "\u0000\u00cf\u0386\u0001\u0000\u0000\u0000\u00d1\u038a\u0001\u0000\u0000"+ + "\u0000\u00d3\u038e\u0001\u0000\u0000\u0000\u00d5\u0392\u0001\u0000\u0000"+ + "\u0000\u00d7\u0396\u0001\u0000\u0000\u0000\u00d9\u039a\u0001\u0000\u0000"+ + "\u0000\u00db\u039f\u0001\u0000\u0000\u0000\u00dd\u03a3\u0001\u0000\u0000"+ + "\u0000\u00df\u03a7\u0001\u0000\u0000\u0000\u00e1\u03ab\u0001\u0000\u0000"+ + "\u0000\u00e3\u03ae\u0001\u0000\u0000\u0000\u00e5\u03b2\u0001\u0000\u0000"+ + "\u0000\u00e7\u03b6\u0001\u0000\u0000\u0000\u00e9\u03ba\u0001\u0000\u0000"+ + "\u0000\u00eb\u03be\u0001\u0000\u0000\u0000\u00ed\u03c3\u0001\u0000\u0000"+ + "\u0000\u00ef\u03c8\u0001\u0000\u0000\u0000\u00f1\u03cd\u0001\u0000\u0000"+ + "\u0000\u00f3\u03d4\u0001\u0000\u0000\u0000\u00f5\u03dd\u0001\u0000\u0000"+ + "\u0000\u00f7\u03e4\u0001\u0000\u0000\u0000\u00f9\u03e8\u0001\u0000\u0000"+ + "\u0000\u00fb\u03ec\u0001\u0000\u0000\u0000\u00fd\u03f0\u0001\u0000\u0000"+ + "\u0000\u00ff\u03f4\u0001\u0000\u0000\u0000\u0101\u03f8\u0001\u0000\u0000"+ + "\u0000\u0103\u03fe\u0001\u0000\u0000\u0000\u0105\u0402\u0001\u0000\u0000"+ + "\u0000\u0107\u0406\u0001\u0000\u0000\u0000\u0109\u040a\u0001\u0000\u0000"+ + "\u0000\u010b\u040e\u0001\u0000\u0000\u0000\u010d\u0412\u0001\u0000\u0000"+ + "\u0000\u010f\u0416\u0001\u0000\u0000\u0000\u0111\u041a\u0001\u0000\u0000"+ + "\u0000\u0113\u041e\u0001\u0000\u0000\u0000\u0115\u0422\u0001\u0000\u0000"+ + "\u0000\u0117\u0427\u0001\u0000\u0000\u0000\u0119\u042b\u0001\u0000\u0000"+ + "\u0000\u011b\u042f\u0001\u0000\u0000\u0000\u011d\u0433\u0001\u0000\u0000"+ + "\u0000\u011f\u0437\u0001\u0000\u0000\u0000\u0121\u043b\u0001\u0000\u0000"+ + "\u0000\u0123\u043f\u0001\u0000\u0000\u0000\u0125\u0444\u0001\u0000\u0000"+ + "\u0000\u0127\u0449\u0001\u0000\u0000\u0000\u0129\u0453\u0001\u0000\u0000"+ + "\u0000\u012b\u0457\u0001\u0000\u0000\u0000\u012d\u045b\u0001\u0000\u0000"+ + "\u0000\u012f\u045f\u0001\u0000\u0000\u0000\u0131\u0464\u0001\u0000\u0000"+ + "\u0000\u0133\u046b\u0001\u0000\u0000\u0000\u0135\u046f\u0001\u0000\u0000"+ + "\u0000\u0137\u0473\u0001\u0000\u0000\u0000\u0139\u0477\u0001\u0000\u0000"+ + "\u0000\u013b\u013c\u0005d\u0000\u0000\u013c\u013d\u0005i\u0000\u0000\u013d"+ + "\u013e\u0005s\u0000\u0000\u013e\u013f\u0005s\u0000\u0000\u013f\u0140\u0005"+ + "e\u0000\u0000\u0140\u0141\u0005c\u0000\u0000\u0141\u0142\u0005t\u0000"+ + "\u0000\u0142\u0143\u0001\u0000\u0000\u0000\u0143\u0144\u0006\u0000\u0000"+ + "\u0000\u0144\f\u0001\u0000\u0000\u0000\u0145\u0146\u0005d\u0000\u0000"+ + "\u0146\u0147\u0005r\u0000\u0000\u0147\u0148\u0005o\u0000\u0000\u0148\u0149"+ + "\u0005p\u0000\u0000\u0149\u014a\u0001\u0000\u0000\u0000\u014a\u014b\u0006"+ + "\u0001\u0001\u0000\u014b\u000e\u0001\u0000\u0000\u0000\u014c\u014d\u0005"+ + "e\u0000\u0000\u014d\u014e\u0005n\u0000\u0000\u014e\u014f\u0005r\u0000"+ + "\u0000\u014f\u0150\u0005i\u0000\u0000\u0150\u0151\u0005c\u0000\u0000\u0151"+ + "\u0152\u0005h\u0000\u0000\u0152\u0153\u0001\u0000\u0000\u0000\u0153\u0154"+ + "\u0006\u0002\u0002\u0000\u0154\u0010\u0001\u0000\u0000\u0000\u0155\u0156"+ + "\u0005e\u0000\u0000\u0156\u0157\u0005v\u0000\u0000\u0157\u0158\u0005a"+ + "\u0000\u0000\u0158\u0159\u0005l\u0000\u0000\u0159\u015a\u0001\u0000\u0000"+ + "\u0000\u015a\u015b\u0006\u0003\u0000\u0000\u015b\u0012\u0001\u0000\u0000"+ + "\u0000\u015c\u015d\u0005e\u0000\u0000\u015d\u015e\u0005x\u0000\u0000\u015e"+ + "\u015f\u0005p\u0000\u0000\u015f\u0160\u0005l\u0000\u0000\u0160\u0161\u0005"+ + "a\u0000\u0000\u0161\u0162\u0005i\u0000\u0000\u0162\u0163\u0005n\u0000"+ + "\u0000\u0163\u0164\u0001\u0000\u0000\u0000\u0164\u0165\u0006\u0004\u0003"+ + "\u0000\u0165\u0014\u0001\u0000\u0000\u0000\u0166\u0167\u0005f\u0000\u0000"+ + "\u0167\u0168\u0005r\u0000\u0000\u0168\u0169\u0005o\u0000\u0000\u0169\u016a"+ + "\u0005m\u0000\u0000\u016a\u016b\u0001\u0000\u0000\u0000\u016b\u016c\u0006"+ + "\u0005\u0004\u0000\u016c\u0016\u0001\u0000\u0000\u0000\u016d\u016e\u0005"+ + "g\u0000\u0000\u016e\u016f\u0005r\u0000\u0000\u016f\u0170\u0005o\u0000"+ + "\u0000\u0170\u0171\u0005k\u0000\u0000\u0171\u0172\u0001\u0000\u0000\u0000"+ + "\u0172\u0173\u0006\u0006\u0000\u0000\u0173\u0018\u0001\u0000\u0000\u0000"+ + "\u0174\u0175\u0005i\u0000\u0000\u0175\u0176\u0005n\u0000\u0000\u0176\u0177"+ + "\u0005l\u0000\u0000\u0177\u0178\u0005i\u0000\u0000\u0178\u0179\u0005n"+ + "\u0000\u0000\u0179\u017a\u0005e\u0000\u0000\u017a\u017b\u0005s\u0000\u0000"+ + "\u017b\u017c\u0005t\u0000\u0000\u017c\u017d\u0005a\u0000\u0000\u017d\u017e"+ + "\u0005t\u0000\u0000\u017e\u017f\u0005s\u0000\u0000\u017f\u0180\u0001\u0000"+ + "\u0000\u0000\u0180\u0181\u0006\u0007\u0000\u0000\u0181\u001a\u0001\u0000"+ + "\u0000\u0000\u0182\u0183\u0005k\u0000\u0000\u0183\u0184\u0005e\u0000\u0000"+ + "\u0184\u0185\u0005e\u0000\u0000\u0185\u0186\u0005p\u0000\u0000\u0186\u0187"+ + "\u0001\u0000\u0000\u0000\u0187\u0188\u0006\b\u0001\u0000\u0188\u001c\u0001"+ + "\u0000\u0000\u0000\u0189\u018a\u0005l\u0000\u0000\u018a\u018b\u0005i\u0000"+ + "\u0000\u018b\u018c\u0005m\u0000\u0000\u018c\u018d\u0005i\u0000\u0000\u018d"+ + "\u018e\u0005t\u0000\u0000\u018e\u018f\u0001\u0000\u0000\u0000\u018f\u0190"+ + "\u0006\t\u0000\u0000\u0190\u001e\u0001\u0000\u0000\u0000\u0191\u0192\u0005"+ + "m\u0000\u0000\u0192\u0193\u0005v\u0000\u0000\u0193\u0194\u0005_\u0000"+ + "\u0000\u0194\u0195\u0005e\u0000\u0000\u0195\u0196\u0005x\u0000\u0000\u0196"+ + "\u0197\u0005p\u0000\u0000\u0197\u0198\u0005a\u0000\u0000\u0198\u0199\u0005"+ + "n\u0000\u0000\u0199\u019a\u0005d\u0000\u0000\u019a\u019b\u0001\u0000\u0000"+ + "\u0000\u019b\u019c\u0006\n\u0005\u0000\u019c \u0001\u0000\u0000\u0000"+ + "\u019d\u019e\u0005r\u0000\u0000\u019e\u019f\u0005e\u0000\u0000\u019f\u01a0"+ + "\u0005n\u0000\u0000\u01a0\u01a1\u0005a\u0000\u0000\u01a1\u01a2\u0005m"+ + "\u0000\u0000\u01a2\u01a3\u0005e\u0000\u0000\u01a3\u01a4\u0001\u0000\u0000"+ + "\u0000\u01a4\u01a5\u0006\u000b\u0006\u0000\u01a5\"\u0001\u0000\u0000\u0000"+ + "\u01a6\u01a7\u0005r\u0000\u0000\u01a7\u01a8\u0005o\u0000\u0000\u01a8\u01a9"+ + "\u0005w\u0000\u0000\u01a9\u01aa\u0001\u0000\u0000\u0000\u01aa\u01ab\u0006"+ + "\f\u0000\u0000\u01ab$\u0001\u0000\u0000\u0000\u01ac\u01ad\u0005s\u0000"+ + "\u0000\u01ad\u01ae\u0005h\u0000\u0000\u01ae\u01af\u0005o\u0000\u0000\u01af"+ + "\u01b0\u0005w\u0000\u0000\u01b0\u01b1\u0001\u0000\u0000\u0000\u01b1\u01b2"+ + "\u0006\r\u0007\u0000\u01b2&\u0001\u0000\u0000\u0000\u01b3\u01b4\u0005"+ + "s\u0000\u0000\u01b4\u01b5\u0005o\u0000\u0000\u01b5\u01b6\u0005r\u0000"+ + "\u0000\u01b6\u01b7\u0005t\u0000\u0000\u01b7\u01b8\u0001\u0000\u0000\u0000"+ + "\u01b8\u01b9\u0006\u000e\u0000\u0000\u01b9(\u0001\u0000\u0000\u0000\u01ba"+ + "\u01bb\u0005s\u0000\u0000\u01bb\u01bc\u0005t\u0000\u0000\u01bc\u01bd\u0005"+ + "a\u0000\u0000\u01bd\u01be\u0005t\u0000\u0000\u01be\u01bf\u0005s\u0000"+ + "\u0000\u01bf\u01c0\u0001\u0000\u0000\u0000\u01c0\u01c1\u0006\u000f\u0000"+ + "\u0000\u01c1*\u0001\u0000\u0000\u0000\u01c2\u01c3\u0005w\u0000\u0000\u01c3"+ + "\u01c4\u0005h\u0000\u0000\u01c4\u01c5\u0005e\u0000\u0000\u01c5\u01c6\u0005"+ + "r\u0000\u0000\u01c6\u01c7\u0005e\u0000\u0000\u01c7\u01c8\u0001\u0000\u0000"+ + "\u0000\u01c8\u01c9\u0006\u0010\u0000\u0000\u01c9,\u0001\u0000\u0000\u0000"+ + "\u01ca\u01cc\b\u0000\u0000\u0000\u01cb\u01ca\u0001\u0000\u0000\u0000\u01cc"+ + "\u01cd\u0001\u0000\u0000\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01cd"+ + "\u01ce\u0001\u0000\u0000\u0000\u01ce\u01cf\u0001\u0000\u0000\u0000\u01cf"+ + "\u01d0\u0006\u0011\u0000\u0000\u01d0.\u0001\u0000\u0000\u0000\u01d1\u01d2"+ + "\u0005/\u0000\u0000\u01d2\u01d3\u0005/\u0000\u0000\u01d3\u01d7\u0001\u0000"+ + "\u0000\u0000\u01d4\u01d6\b\u0001\u0000\u0000\u01d5\u01d4\u0001\u0000\u0000"+ + "\u0000\u01d6\u01d9\u0001\u0000\u0000\u0000\u01d7\u01d5\u0001\u0000\u0000"+ + "\u0000\u01d7\u01d8\u0001\u0000\u0000\u0000\u01d8\u01db\u0001\u0000\u0000"+ + "\u0000\u01d9\u01d7\u0001\u0000\u0000\u0000\u01da\u01dc\u0005\r\u0000\u0000"+ + "\u01db\u01da\u0001\u0000\u0000\u0000\u01db\u01dc\u0001\u0000\u0000\u0000"+ + "\u01dc\u01de\u0001\u0000\u0000\u0000\u01dd\u01df\u0005\n\u0000\u0000\u01de"+ + "\u01dd\u0001\u0000\u0000\u0000\u01de\u01df\u0001\u0000\u0000\u0000\u01df"+ + "\u01e0\u0001\u0000\u0000\u0000\u01e0\u01e1\u0006\u0012\b\u0000\u01e10"+ + "\u0001\u0000\u0000\u0000\u01e2\u01e3\u0005/\u0000\u0000\u01e3\u01e4\u0005"+ + "*\u0000\u0000\u01e4\u01e9\u0001\u0000\u0000\u0000\u01e5\u01e8\u00031\u0013"+ + "\u0000\u01e6\u01e8\t\u0000\u0000\u0000\u01e7\u01e5\u0001\u0000\u0000\u0000"+ + "\u01e7\u01e6\u0001\u0000\u0000\u0000\u01e8\u01eb\u0001\u0000\u0000\u0000"+ + "\u01e9\u01ea\u0001\u0000\u0000\u0000\u01e9\u01e7\u0001\u0000\u0000\u0000"+ + "\u01ea\u01ec\u0001\u0000\u0000\u0000\u01eb\u01e9\u0001\u0000\u0000\u0000"+ + "\u01ec\u01ed\u0005*\u0000\u0000\u01ed\u01ee\u0005/\u0000\u0000\u01ee\u01ef"+ + "\u0001\u0000\u0000\u0000\u01ef\u01f0\u0006\u0013\b\u0000\u01f02\u0001"+ + "\u0000\u0000\u0000\u01f1\u01f3\u0007\u0002\u0000\u0000\u01f2\u01f1\u0001"+ + "\u0000\u0000\u0000\u01f3\u01f4\u0001\u0000\u0000\u0000\u01f4\u01f2\u0001"+ + "\u0000\u0000\u0000\u01f4\u01f5\u0001\u0000\u0000\u0000\u01f5\u01f6\u0001"+ + "\u0000\u0000\u0000\u01f6\u01f7\u0006\u0014\b\u0000\u01f74\u0001\u0000"+ + "\u0000\u0000\u01f8\u01f9\u0003\u009fJ\u0000\u01f9\u01fa\u0001\u0000\u0000"+ + "\u0000\u01fa\u01fb\u0006\u0015\t\u0000\u01fb\u01fc\u0006\u0015\n\u0000"+ + "\u01fc6\u0001\u0000\u0000\u0000\u01fd\u01fe\u0003?\u001a\u0000\u01fe\u01ff"+ + "\u0001\u0000\u0000\u0000\u01ff\u0200\u0006\u0016\u000b\u0000\u0200\u0201"+ + "\u0006\u0016\f\u0000\u02018\u0001\u0000\u0000\u0000\u0202\u0203\u0003"+ + "3\u0014\u0000\u0203\u0204\u0001\u0000\u0000\u0000\u0204\u0205\u0006\u0017"+ + "\b\u0000\u0205:\u0001\u0000\u0000\u0000\u0206\u0207\u0003/\u0012\u0000"+ + "\u0207\u0208\u0001\u0000\u0000\u0000\u0208\u0209\u0006\u0018\b\u0000\u0209"+ + "<\u0001\u0000\u0000\u0000\u020a\u020b\u00031\u0013\u0000\u020b\u020c\u0001"+ + "\u0000\u0000\u0000\u020c\u020d\u0006\u0019\b\u0000\u020d>\u0001\u0000"+ + "\u0000\u0000\u020e\u020f\u0005|\u0000\u0000\u020f\u0210\u0001\u0000\u0000"+ + "\u0000\u0210\u0211\u0006\u001a\f\u0000\u0211@\u0001\u0000\u0000\u0000"+ + "\u0212\u0213\u0007\u0003\u0000\u0000\u0213B\u0001\u0000\u0000\u0000\u0214"+ + "\u0215\u0007\u0004\u0000\u0000\u0215D\u0001\u0000\u0000\u0000\u0216\u0217"+ + "\u0005\\\u0000\u0000\u0217\u0218\u0007\u0005\u0000\u0000\u0218F\u0001"+ + "\u0000\u0000\u0000\u0219\u021a\b\u0006\u0000\u0000\u021aH\u0001\u0000"+ + "\u0000\u0000\u021b\u021d\u0007\u0007\u0000\u0000\u021c\u021e\u0007\b\u0000"+ + "\u0000\u021d\u021c\u0001\u0000\u0000\u0000\u021d\u021e\u0001\u0000\u0000"+ + "\u0000\u021e\u0220\u0001\u0000\u0000\u0000\u021f\u0221\u0003A\u001b\u0000"+ + "\u0220\u021f\u0001\u0000\u0000\u0000\u0221\u0222\u0001\u0000\u0000\u0000"+ + "\u0222\u0220\u0001\u0000\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000"+ + "\u0223J\u0001\u0000\u0000\u0000\u0224\u0225\u0005@\u0000\u0000\u0225L"+ + "\u0001\u0000\u0000\u0000\u0226\u0227\u0005`\u0000\u0000\u0227N\u0001\u0000"+ + "\u0000\u0000\u0228\u022c\b\t\u0000\u0000\u0229\u022a\u0005`\u0000\u0000"+ + "\u022a\u022c\u0005`\u0000\u0000\u022b\u0228\u0001\u0000\u0000\u0000\u022b"+ + "\u0229\u0001\u0000\u0000\u0000\u022cP\u0001\u0000\u0000\u0000\u022d\u022e"+ + "\u0005_\u0000\u0000\u022eR\u0001\u0000\u0000\u0000\u022f\u0233\u0003C"+ + "\u001c\u0000\u0230\u0233\u0003A\u001b\u0000\u0231\u0233\u0003Q#\u0000"+ + "\u0232\u022f\u0001\u0000\u0000\u0000\u0232\u0230\u0001\u0000\u0000\u0000"+ + "\u0232\u0231\u0001\u0000\u0000\u0000\u0233T\u0001\u0000\u0000\u0000\u0234"+ + "\u0239\u0005\"\u0000\u0000\u0235\u0238\u0003E\u001d\u0000\u0236\u0238"+ + "\u0003G\u001e\u0000\u0237\u0235\u0001\u0000\u0000\u0000\u0237\u0236\u0001"+ + "\u0000\u0000\u0000\u0238\u023b\u0001\u0000\u0000\u0000\u0239\u0237\u0001"+ + "\u0000\u0000\u0000\u0239\u023a\u0001\u0000\u0000\u0000\u023a\u023c\u0001"+ + "\u0000\u0000\u0000\u023b\u0239\u0001\u0000\u0000\u0000\u023c\u0252\u0005"+ + "\"\u0000\u0000\u023d\u023e\u0005\"\u0000\u0000\u023e\u023f\u0005\"\u0000"+ + "\u0000\u023f\u0240\u0005\"\u0000\u0000\u0240\u0244\u0001\u0000\u0000\u0000"+ + "\u0241\u0243\b\u0001\u0000\u0000\u0242\u0241\u0001\u0000\u0000\u0000\u0243"+ + "\u0246\u0001\u0000\u0000\u0000\u0244\u0245\u0001\u0000\u0000\u0000\u0244"+ + "\u0242\u0001\u0000\u0000\u0000\u0245\u0247\u0001\u0000\u0000\u0000\u0246"+ + "\u0244\u0001\u0000\u0000\u0000\u0247\u0248\u0005\"\u0000\u0000\u0248\u0249"+ + "\u0005\"\u0000\u0000\u0249\u024a\u0005\"\u0000\u0000\u024a\u024c\u0001"+ + "\u0000\u0000\u0000\u024b\u024d\u0005\"\u0000\u0000\u024c\u024b\u0001\u0000"+ + "\u0000\u0000\u024c\u024d\u0001\u0000\u0000\u0000\u024d\u024f\u0001\u0000"+ + "\u0000\u0000\u024e\u0250\u0005\"\u0000\u0000\u024f\u024e\u0001\u0000\u0000"+ + "\u0000\u024f\u0250\u0001\u0000\u0000\u0000\u0250\u0252\u0001\u0000\u0000"+ + "\u0000\u0251\u0234\u0001\u0000\u0000\u0000\u0251\u023d\u0001\u0000\u0000"+ + "\u0000\u0252V\u0001\u0000\u0000\u0000\u0253\u0255\u0003A\u001b\u0000\u0254"+ + "\u0253\u0001\u0000\u0000\u0000\u0255\u0256\u0001\u0000\u0000\u0000\u0256"+ + "\u0254\u0001\u0000\u0000\u0000\u0256\u0257\u0001\u0000\u0000\u0000\u0257"+ + "X\u0001\u0000\u0000\u0000\u0258\u025a\u0003A\u001b\u0000\u0259\u0258\u0001"+ + "\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000\u0000\u025b\u0259\u0001"+ + "\u0000\u0000\u0000\u025b\u025c\u0001\u0000\u0000\u0000\u025c\u025d\u0001"+ + "\u0000\u0000\u0000\u025d\u0261\u0003g.\u0000\u025e\u0260\u0003A\u001b"+ + "\u0000\u025f\u025e\u0001\u0000\u0000\u0000\u0260\u0263\u0001\u0000\u0000"+ + "\u0000\u0261\u025f\u0001\u0000\u0000\u0000\u0261\u0262\u0001\u0000\u0000"+ + "\u0000\u0262\u0283\u0001\u0000\u0000\u0000\u0263\u0261\u0001\u0000\u0000"+ + "\u0000\u0264\u0266\u0003g.\u0000\u0265\u0267\u0003A\u001b\u0000\u0266"+ + "\u0265\u0001\u0000\u0000\u0000\u0267\u0268\u0001\u0000\u0000\u0000\u0268"+ + "\u0266\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000\u0000\u0269"+ + "\u0283\u0001\u0000\u0000\u0000\u026a\u026c\u0003A\u001b\u0000\u026b\u026a"+ + "\u0001\u0000\u0000\u0000\u026c\u026d\u0001\u0000\u0000\u0000\u026d\u026b"+ + "\u0001\u0000\u0000\u0000\u026d\u026e\u0001\u0000\u0000\u0000\u026e\u0276"+ + "\u0001\u0000\u0000\u0000\u026f\u0273\u0003g.\u0000\u0270\u0272\u0003A"+ + "\u001b\u0000\u0271\u0270\u0001\u0000\u0000\u0000\u0272\u0275\u0001\u0000"+ + "\u0000\u0000\u0273\u0271\u0001\u0000\u0000\u0000\u0273\u0274\u0001\u0000"+ + "\u0000\u0000\u0274\u0277\u0001\u0000\u0000\u0000\u0275\u0273\u0001\u0000"+ + "\u0000\u0000\u0276\u026f\u0001\u0000\u0000\u0000\u0276\u0277\u0001\u0000"+ + "\u0000\u0000\u0277\u0278\u0001\u0000\u0000\u0000\u0278\u0279\u0003I\u001f"+ + "\u0000\u0279\u0283\u0001\u0000\u0000\u0000\u027a\u027c\u0003g.\u0000\u027b"+ + "\u027d\u0003A\u001b\u0000\u027c\u027b\u0001\u0000\u0000\u0000\u027d\u027e"+ + "\u0001\u0000\u0000\u0000\u027e\u027c\u0001\u0000\u0000\u0000\u027e\u027f"+ + "\u0001\u0000\u0000\u0000\u027f\u0280\u0001\u0000\u0000\u0000\u0280\u0281"+ + "\u0003I\u001f\u0000\u0281\u0283\u0001\u0000\u0000\u0000\u0282\u0259\u0001"+ + "\u0000\u0000\u0000\u0282\u0264\u0001\u0000\u0000\u0000\u0282\u026b\u0001"+ + "\u0000\u0000\u0000\u0282\u027a\u0001\u0000\u0000\u0000\u0283Z\u0001\u0000"+ + "\u0000\u0000\u0284\u0285\u0005b\u0000\u0000\u0285\u0286\u0005y\u0000\u0000"+ + "\u0286\\\u0001\u0000\u0000\u0000\u0287\u0288\u0005a\u0000\u0000\u0288"+ + "\u0289\u0005n\u0000\u0000\u0289\u028a\u0005d\u0000\u0000\u028a^\u0001"+ + "\u0000\u0000\u0000\u028b\u028c\u0005a\u0000\u0000\u028c\u028d\u0005s\u0000"+ + "\u0000\u028d\u028e\u0005c\u0000\u0000\u028e`\u0001\u0000\u0000\u0000\u028f"+ + "\u0290\u0005=\u0000\u0000\u0290b\u0001\u0000\u0000\u0000\u0291\u0292\u0005"+ + ",\u0000\u0000\u0292d\u0001\u0000\u0000\u0000\u0293\u0294\u0005d\u0000"+ + "\u0000\u0294\u0295\u0005e\u0000\u0000\u0295\u0296\u0005s\u0000\u0000\u0296"+ + "\u0297\u0005c\u0000\u0000\u0297f\u0001\u0000\u0000\u0000\u0298\u0299\u0005"+ + ".\u0000\u0000\u0299h\u0001\u0000\u0000\u0000\u029a\u029b\u0005f\u0000"+ + "\u0000\u029b\u029c\u0005a\u0000\u0000\u029c\u029d\u0005l\u0000\u0000\u029d"+ + "\u029e\u0005s\u0000\u0000\u029e\u029f\u0005e\u0000\u0000\u029fj\u0001"+ + "\u0000\u0000\u0000\u02a0\u02a1\u0005f\u0000\u0000\u02a1\u02a2\u0005i\u0000"+ + "\u0000\u02a2\u02a3\u0005r\u0000\u0000\u02a3\u02a4\u0005s\u0000\u0000\u02a4"+ + "\u02a5\u0005t\u0000\u0000\u02a5l\u0001\u0000\u0000\u0000\u02a6\u02a7\u0005"+ + "l\u0000\u0000\u02a7\u02a8\u0005a\u0000\u0000\u02a8\u02a9\u0005s\u0000"+ + "\u0000\u02a9\u02aa\u0005t\u0000\u0000\u02aan\u0001\u0000\u0000\u0000\u02ab"+ + "\u02ac\u0005(\u0000\u0000\u02acp\u0001\u0000\u0000\u0000\u02ad\u02ae\u0005"+ + "i\u0000\u0000\u02ae\u02af\u0005n\u0000\u0000\u02afr\u0001\u0000\u0000"+ + "\u0000\u02b0\u02b1\u0005i\u0000\u0000\u02b1\u02b2\u0005s\u0000\u0000\u02b2"+ + "t\u0001\u0000\u0000\u0000\u02b3\u02b4\u0005l\u0000\u0000\u02b4\u02b5\u0005"+ + "i\u0000\u0000\u02b5\u02b6\u0005k\u0000\u0000\u02b6\u02b7\u0005e\u0000"+ + "\u0000\u02b7v\u0001\u0000\u0000\u0000\u02b8\u02b9\u0005n\u0000\u0000\u02b9"+ + "\u02ba\u0005o\u0000\u0000\u02ba\u02bb\u0005t\u0000\u0000\u02bbx\u0001"+ + "\u0000\u0000\u0000\u02bc\u02bd\u0005n\u0000\u0000\u02bd\u02be\u0005u\u0000"+ + "\u0000\u02be\u02bf\u0005l\u0000\u0000\u02bf\u02c0\u0005l\u0000\u0000\u02c0"+ + "z\u0001\u0000\u0000\u0000\u02c1\u02c2\u0005n\u0000\u0000\u02c2\u02c3\u0005"+ + "u\u0000\u0000\u02c3\u02c4\u0005l\u0000\u0000\u02c4\u02c5\u0005l\u0000"+ + "\u0000\u02c5\u02c6\u0005s\u0000\u0000\u02c6|\u0001\u0000\u0000\u0000\u02c7"+ + "\u02c8\u0005o\u0000\u0000\u02c8\u02c9\u0005r\u0000\u0000\u02c9~\u0001"+ + "\u0000\u0000\u0000\u02ca\u02cb\u0005?\u0000\u0000\u02cb\u0080\u0001\u0000"+ + "\u0000\u0000\u02cc\u02cd\u0005r\u0000\u0000\u02cd\u02ce\u0005l\u0000\u0000"+ + "\u02ce\u02cf\u0005i\u0000\u0000\u02cf\u02d0\u0005k\u0000\u0000\u02d0\u02d1"+ + "\u0005e\u0000\u0000\u02d1\u0082\u0001\u0000\u0000\u0000\u02d2\u02d3\u0005"+ + ")\u0000\u0000\u02d3\u0084\u0001\u0000\u0000\u0000\u02d4\u02d5\u0005t\u0000"+ + "\u0000\u02d5\u02d6\u0005r\u0000\u0000\u02d6\u02d7\u0005u\u0000\u0000\u02d7"+ + "\u02d8\u0005e\u0000\u0000\u02d8\u0086\u0001\u0000\u0000\u0000\u02d9\u02da"+ + "\u0005=\u0000\u0000\u02da\u02db\u0005=\u0000\u0000\u02db\u0088\u0001\u0000"+ + "\u0000\u0000\u02dc\u02dd\u0005=\u0000\u0000\u02dd\u02de\u0005~\u0000\u0000"+ + "\u02de\u008a\u0001\u0000\u0000\u0000\u02df\u02e0\u0005!\u0000\u0000\u02e0"+ + "\u02e1\u0005=\u0000\u0000\u02e1\u008c\u0001\u0000\u0000\u0000\u02e2\u02e3"+ + "\u0005<\u0000\u0000\u02e3\u008e\u0001\u0000\u0000\u0000\u02e4\u02e5\u0005"+ + "<\u0000\u0000\u02e5\u02e6\u0005=\u0000\u0000\u02e6\u0090\u0001\u0000\u0000"+ + "\u0000\u02e7\u02e8\u0005>\u0000\u0000\u02e8\u0092\u0001\u0000\u0000\u0000"+ + "\u02e9\u02ea\u0005>\u0000\u0000\u02ea\u02eb\u0005=\u0000\u0000\u02eb\u0094"+ + "\u0001\u0000\u0000\u0000\u02ec\u02ed\u0005+\u0000\u0000\u02ed\u0096\u0001"+ + "\u0000\u0000\u0000\u02ee\u02ef\u0005-\u0000\u0000\u02ef\u0098\u0001\u0000"+ + "\u0000\u0000\u02f0\u02f1\u0005*\u0000\u0000\u02f1\u009a\u0001\u0000\u0000"+ + "\u0000\u02f2\u02f3\u0005/\u0000\u0000\u02f3\u009c\u0001\u0000\u0000\u0000"+ + "\u02f4\u02f5\u0005%\u0000\u0000\u02f5\u009e\u0001\u0000\u0000\u0000\u02f6"+ + "\u02f7\u0005[\u0000\u0000\u02f7\u02f8\u0001\u0000\u0000\u0000\u02f8\u02f9"+ + "\u0006J\u0000\u0000\u02f9\u02fa\u0006J\u0000\u0000\u02fa\u00a0\u0001\u0000"+ + "\u0000\u0000\u02fb\u02fc\u0005]\u0000\u0000\u02fc\u02fd\u0001\u0000\u0000"+ + "\u0000\u02fd\u02fe\u0006K\f\u0000\u02fe\u02ff\u0006K\f\u0000\u02ff\u00a2"+ + "\u0001\u0000\u0000\u0000\u0300\u0304\u0003C\u001c\u0000\u0301\u0303\u0003"+ + "S$\u0000\u0302\u0301\u0001\u0000\u0000\u0000\u0303\u0306\u0001\u0000\u0000"+ + "\u0000\u0304\u0302\u0001\u0000\u0000\u0000\u0304\u0305\u0001\u0000\u0000"+ + "\u0000\u0305\u0311\u0001\u0000\u0000\u0000\u0306\u0304\u0001\u0000\u0000"+ + "\u0000\u0307\u030a\u0003Q#\u0000\u0308\u030a\u0003K \u0000\u0309\u0307"+ + "\u0001\u0000\u0000\u0000\u0309\u0308\u0001\u0000\u0000\u0000\u030a\u030c"+ + "\u0001\u0000\u0000\u0000\u030b\u030d\u0003S$\u0000\u030c\u030b\u0001\u0000"+ + "\u0000\u0000\u030d\u030e\u0001\u0000\u0000\u0000\u030e\u030c\u0001\u0000"+ + "\u0000\u0000\u030e\u030f\u0001\u0000\u0000\u0000\u030f\u0311\u0001\u0000"+ + "\u0000\u0000\u0310\u0300\u0001\u0000\u0000\u0000\u0310\u0309\u0001\u0000"+ + "\u0000\u0000\u0311\u00a4\u0001\u0000\u0000\u0000\u0312\u0314\u0003M!\u0000"+ + "\u0313\u0315\u0003O\"\u0000\u0314\u0313\u0001\u0000\u0000\u0000\u0315"+ + "\u0316\u0001\u0000\u0000\u0000\u0316\u0314\u0001\u0000\u0000\u0000\u0316"+ + "\u0317\u0001\u0000\u0000\u0000\u0317\u0318\u0001\u0000\u0000\u0000\u0318"+ + "\u0319\u0003M!\u0000\u0319\u00a6\u0001\u0000\u0000\u0000\u031a\u031b\u0003"+ + "\u00a5M\u0000\u031b\u00a8\u0001\u0000\u0000\u0000\u031c\u031d\u0003/\u0012"+ + "\u0000\u031d\u031e\u0001\u0000\u0000\u0000\u031e\u031f\u0006O\b\u0000"+ + "\u031f\u00aa\u0001\u0000\u0000\u0000\u0320\u0321\u00031\u0013\u0000\u0321"+ + "\u0322\u0001\u0000\u0000\u0000\u0322\u0323\u0006P\b\u0000\u0323\u00ac"+ + "\u0001\u0000\u0000\u0000\u0324\u0325\u00033\u0014\u0000\u0325\u0326\u0001"+ + "\u0000\u0000\u0000\u0326\u0327\u0006Q\b\u0000\u0327\u00ae\u0001\u0000"+ + "\u0000\u0000\u0328\u0329\u0003?\u001a\u0000\u0329\u032a\u0001\u0000\u0000"+ + "\u0000\u032a\u032b\u0006R\u000b\u0000\u032b\u032c\u0006R\f\u0000\u032c"+ + "\u00b0\u0001\u0000\u0000\u0000\u032d\u032e\u0003\u009fJ\u0000\u032e\u032f"+ + "\u0001\u0000\u0000\u0000\u032f\u0330\u0006S\t\u0000\u0330\u00b2\u0001"+ + "\u0000\u0000\u0000\u0331\u0332\u0003\u00a1K\u0000\u0332\u0333\u0001\u0000"+ + "\u0000\u0000\u0333\u0334\u0006T\r\u0000\u0334\u00b4\u0001\u0000\u0000"+ + "\u0000\u0335\u0336\u0003c,\u0000\u0336\u0337\u0001\u0000\u0000\u0000\u0337"+ + "\u0338\u0006U\u000e\u0000\u0338\u00b6\u0001\u0000\u0000\u0000\u0339\u033a"+ + "\u0003a+\u0000\u033a\u033b\u0001\u0000\u0000\u0000\u033b\u033c\u0006V"+ + "\u000f\u0000\u033c\u00b8\u0001\u0000\u0000\u0000\u033d\u033e\u0005m\u0000"+ "\u0000\u033e\u033f\u0005e\u0000\u0000\u033f\u0340\u0005t\u0000\u0000\u0340"+ "\u0341\u0005a\u0000\u0000\u0341\u0342\u0005d\u0000\u0000\u0342\u0343\u0005"+ "a\u0000\u0000\u0343\u0344\u0005t\u0000\u0000\u0344\u0345\u0005a\u0000"+ - "\u0000\u0345\u00b8\u0001\u0000\u0000\u0000\u0346\u034a\b\n\u0000\u0000"+ + "\u0000\u0345\u00ba\u0001\u0000\u0000\u0000\u0346\u034a\b\n\u0000\u0000"+ "\u0347\u0348\u0005/\u0000\u0000\u0348\u034a\b\u000b\u0000\u0000\u0349"+ "\u0346\u0001\u0000\u0000\u0000\u0349\u0347\u0001\u0000\u0000\u0000\u034a"+ - "\u00ba\u0001\u0000\u0000\u0000\u034b\u034d\u0003\u00b9W\u0000\u034c\u034b"+ + "\u00bc\u0001\u0000\u0000\u0000\u034b\u034d\u0003\u00bbX\u0000\u034c\u034b"+ "\u0001\u0000\u0000\u0000\u034d\u034e\u0001\u0000\u0000\u0000\u034e\u034c"+ - "\u0001\u0000\u0000\u0000\u034e\u034f\u0001\u0000\u0000\u0000\u034f\u00bc"+ - "\u0001\u0000\u0000\u0000\u0350\u0351\u0003\u00a5M\u0000\u0351\u0352\u0001"+ - "\u0000\u0000\u0000\u0352\u0353\u0006Y\u0010\u0000\u0353\u00be\u0001\u0000"+ + "\u0001\u0000\u0000\u0000\u034e\u034f\u0001\u0000\u0000\u0000\u034f\u00be"+ + "\u0001\u0000\u0000\u0000\u0350\u0351\u0003\u00a7N\u0000\u0351\u0352\u0001"+ + "\u0000\u0000\u0000\u0352\u0353\u0006Z\u0010\u0000\u0353\u00c0\u0001\u0000"+ "\u0000\u0000\u0354\u0355\u0003/\u0012\u0000\u0355\u0356\u0001\u0000\u0000"+ - "\u0000\u0356\u0357\u0006Z\b\u0000\u0357\u00c0\u0001\u0000\u0000\u0000"+ + "\u0000\u0356\u0357\u0006[\b\u0000\u0357\u00c2\u0001\u0000\u0000\u0000"+ "\u0358\u0359\u00031\u0013\u0000\u0359\u035a\u0001\u0000\u0000\u0000\u035a"+ - "\u035b\u0006[\b\u0000\u035b\u00c2\u0001\u0000\u0000\u0000\u035c\u035d"+ + "\u035b\u0006\\\b\u0000\u035b\u00c4\u0001\u0000\u0000\u0000\u035c\u035d"+ "\u00033\u0014\u0000\u035d\u035e\u0001\u0000\u0000\u0000\u035e\u035f\u0006"+ - "\\\b\u0000\u035f\u00c4\u0001\u0000\u0000\u0000\u0360\u0361\u0003?\u001a"+ - "\u0000\u0361\u0362\u0001\u0000\u0000\u0000\u0362\u0363\u0006]\u000b\u0000"+ - "\u0363\u0364\u0006]\f\u0000\u0364\u00c6\u0001\u0000\u0000\u0000\u0365"+ + "]\b\u0000\u035f\u00c6\u0001\u0000\u0000\u0000\u0360\u0361\u0003?\u001a"+ + "\u0000\u0361\u0362\u0001\u0000\u0000\u0000\u0362\u0363\u0006^\u000b\u0000"+ + "\u0363\u0364\u0006^\f\u0000\u0364\u00c8\u0001\u0000\u0000\u0000\u0365"+ "\u0366\u0003g.\u0000\u0366\u0367\u0001\u0000\u0000\u0000\u0367\u0368\u0006"+ - "^\u0011\u0000\u0368\u00c8\u0001\u0000\u0000\u0000\u0369\u036a\u0003c,"+ - "\u0000\u036a\u036b\u0001\u0000\u0000\u0000\u036b\u036c\u0006_\u000e\u0000"+ - "\u036c\u00ca\u0001\u0000\u0000\u0000\u036d\u0372\u0003C\u001c\u0000\u036e"+ + "_\u0011\u0000\u0368\u00ca\u0001\u0000\u0000\u0000\u0369\u036a\u0003c,"+ + "\u0000\u036a\u036b\u0001\u0000\u0000\u0000\u036b\u036c\u0006`\u000e\u0000"+ + "\u036c\u00cc\u0001\u0000\u0000\u0000\u036d\u0372\u0003C\u001c\u0000\u036e"+ "\u0372\u0003A\u001b\u0000\u036f\u0372\u0003Q#\u0000\u0370\u0372\u0003"+ "\u0099G\u0000\u0371\u036d\u0001\u0000\u0000\u0000\u0371\u036e\u0001\u0000"+ "\u0000\u0000\u0371\u036f\u0001\u0000\u0000\u0000\u0371\u0370\u0001\u0000"+ - "\u0000\u0000\u0372\u00cc\u0001\u0000\u0000\u0000\u0373\u0376\u0003C\u001c"+ + "\u0000\u0000\u0372\u00ce\u0001\u0000\u0000\u0000\u0373\u0376\u0003C\u001c"+ "\u0000\u0374\u0376\u0003\u0099G\u0000\u0375\u0373\u0001\u0000\u0000\u0000"+ "\u0375\u0374\u0001\u0000\u0000\u0000\u0376\u037a\u0001\u0000\u0000\u0000"+ - "\u0377\u0379\u0003\u00cb`\u0000\u0378\u0377\u0001\u0000\u0000\u0000\u0379"+ + "\u0377\u0379\u0003\u00cda\u0000\u0378\u0377\u0001\u0000\u0000\u0000\u0379"+ "\u037c\u0001\u0000\u0000\u0000\u037a\u0378\u0001\u0000\u0000\u0000\u037a"+ "\u037b\u0001\u0000\u0000\u0000\u037b\u0387\u0001\u0000\u0000\u0000\u037c"+ "\u037a\u0001\u0000\u0000\u0000\u037d\u0380\u0003Q#\u0000\u037e\u0380\u0003"+ "K \u0000\u037f\u037d\u0001\u0000\u0000\u0000\u037f\u037e\u0001\u0000\u0000"+ - "\u0000\u0380\u0382\u0001\u0000\u0000\u0000\u0381\u0383\u0003\u00cb`\u0000"+ + "\u0000\u0380\u0382\u0001\u0000\u0000\u0000\u0381\u0383\u0003\u00cda\u0000"+ "\u0382\u0381\u0001\u0000\u0000\u0000\u0383\u0384\u0001\u0000\u0000\u0000"+ "\u0384\u0382\u0001\u0000\u0000\u0000\u0384\u0385\u0001\u0000\u0000\u0000"+ "\u0385\u0387\u0001\u0000\u0000\u0000\u0386\u0375\u0001\u0000\u0000\u0000"+ - "\u0386\u037f\u0001\u0000\u0000\u0000\u0387\u00ce\u0001\u0000\u0000\u0000"+ - "\u0388\u0389\u0003\u00cda\u0000\u0389\u038a\u0001\u0000\u0000\u0000\u038a"+ - "\u038b\u0006b\u0012\u0000\u038b\u00d0\u0001\u0000\u0000\u0000\u038c\u038d"+ - "\u0003\u00a5M\u0000\u038d\u038e\u0001\u0000\u0000\u0000\u038e\u038f\u0006"+ - "c\u0010\u0000\u038f\u00d2\u0001\u0000\u0000\u0000\u0390\u0391\u0003/\u0012"+ - "\u0000\u0391\u0392\u0001\u0000\u0000\u0000\u0392\u0393\u0006d\b\u0000"+ - "\u0393\u00d4\u0001\u0000\u0000\u0000\u0394\u0395\u00031\u0013\u0000\u0395"+ - "\u0396\u0001\u0000\u0000\u0000\u0396\u0397\u0006e\b\u0000\u0397\u00d6"+ - "\u0001\u0000\u0000\u0000\u0398\u0399\u00033\u0014\u0000\u0399\u039a\u0001"+ - "\u0000\u0000\u0000\u039a\u039b\u0006f\b\u0000\u039b\u00d8\u0001\u0000"+ - "\u0000\u0000\u039c\u039d\u0003?\u001a\u0000\u039d\u039e\u0001\u0000\u0000"+ - "\u0000\u039e\u039f\u0006g\u000b\u0000\u039f\u03a0\u0006g\f\u0000\u03a0"+ - "\u00da\u0001\u0000\u0000\u0000\u03a1\u03a2\u0003a+\u0000\u03a2\u03a3\u0001"+ - "\u0000\u0000\u0000\u03a3\u03a4\u0006h\u000f\u0000\u03a4\u00dc\u0001\u0000"+ - "\u0000\u0000\u03a5\u03a6\u0003c,\u0000\u03a6\u03a7\u0001\u0000\u0000\u0000"+ - "\u03a7\u03a8\u0006i\u000e\u0000\u03a8\u00de\u0001\u0000\u0000\u0000\u03a9"+ - "\u03aa\u0003g.\u0000\u03aa\u03ab\u0001\u0000\u0000\u0000\u03ab\u03ac\u0006"+ - "j\u0011\u0000\u03ac\u00e0\u0001\u0000\u0000\u0000\u03ad\u03ae\u0005a\u0000"+ - "\u0000\u03ae\u03af\u0005s\u0000\u0000\u03af\u00e2\u0001\u0000\u0000\u0000"+ - "\u03b0\u03b1\u0003\u00a5M\u0000\u03b1\u03b2\u0001\u0000\u0000\u0000\u03b2"+ - "\u03b3\u0006l\u0010\u0000\u03b3\u00e4\u0001\u0000\u0000\u0000\u03b4\u03b5"+ - "\u0003\u00cda\u0000\u03b5\u03b6\u0001\u0000\u0000\u0000\u03b6\u03b7\u0006"+ - "m\u0012\u0000\u03b7\u00e6\u0001\u0000\u0000\u0000\u03b8\u03b9\u0003/\u0012"+ - "\u0000\u03b9\u03ba\u0001\u0000\u0000\u0000\u03ba\u03bb\u0006n\b\u0000"+ - "\u03bb\u00e8\u0001\u0000\u0000\u0000\u03bc\u03bd\u00031\u0013\u0000\u03bd"+ - "\u03be\u0001\u0000\u0000\u0000\u03be\u03bf\u0006o\b\u0000\u03bf\u00ea"+ - "\u0001\u0000\u0000\u0000\u03c0\u03c1\u00033\u0014\u0000\u03c1\u03c2\u0001"+ - "\u0000\u0000\u0000\u03c2\u03c3\u0006p\b\u0000\u03c3\u00ec\u0001\u0000"+ - "\u0000\u0000\u03c4\u03c5\u0003?\u001a\u0000\u03c5\u03c6\u0001\u0000\u0000"+ - "\u0000\u03c6\u03c7\u0006q\u000b\u0000\u03c7\u03c8\u0006q\f\u0000\u03c8"+ - "\u00ee\u0001\u0000\u0000\u0000\u03c9\u03ca\u0003\u009fJ\u0000\u03ca\u03cb"+ - "\u0001\u0000\u0000\u0000\u03cb\u03cc\u0006r\t\u0000\u03cc\u03cd\u0006"+ - "r\u0013\u0000\u03cd\u00f0\u0001\u0000\u0000\u0000\u03ce\u03cf\u0005o\u0000"+ - "\u0000\u03cf\u03d0\u0005n\u0000\u0000\u03d0\u03d1\u0001\u0000\u0000\u0000"+ - "\u03d1\u03d2\u0006s\u0014\u0000\u03d2\u00f2\u0001\u0000\u0000\u0000\u03d3"+ - "\u03d4\u0005w\u0000\u0000\u03d4\u03d5\u0005i\u0000\u0000\u03d5\u03d6\u0005"+ - "t\u0000\u0000\u03d6\u03d7\u0005h\u0000\u0000\u03d7\u03d8\u0001\u0000\u0000"+ - "\u0000\u03d8\u03d9\u0006t\u0014\u0000\u03d9\u00f4\u0001\u0000\u0000\u0000"+ - "\u03da\u03db\b\f\u0000\u0000\u03db\u00f6\u0001\u0000\u0000\u0000\u03dc"+ - "\u03de\u0003\u00f5u\u0000\u03dd\u03dc\u0001\u0000\u0000\u0000\u03de\u03df"+ - "\u0001\u0000\u0000\u0000\u03df\u03dd\u0001\u0000\u0000\u0000\u03df\u03e0"+ - "\u0001\u0000\u0000\u0000\u03e0\u03e1\u0001\u0000\u0000\u0000\u03e1\u03e2"+ - "\u0003\u0133\u0094\u0000\u03e2\u03e4\u0001\u0000\u0000\u0000\u03e3\u03dd"+ - "\u0001\u0000\u0000\u0000\u03e3\u03e4\u0001\u0000\u0000\u0000\u03e4\u03e6"+ - "\u0001\u0000\u0000\u0000\u03e5\u03e7\u0003\u00f5u\u0000\u03e6\u03e5\u0001"+ - "\u0000\u0000\u0000\u03e7\u03e8\u0001\u0000\u0000\u0000\u03e8\u03e6\u0001"+ - "\u0000\u0000\u0000\u03e8\u03e9\u0001\u0000\u0000\u0000\u03e9\u00f8\u0001"+ - "\u0000\u0000\u0000\u03ea\u03eb\u0003\u00a5M\u0000\u03eb\u03ec\u0001\u0000"+ - "\u0000\u0000\u03ec\u03ed\u0006w\u0010\u0000\u03ed\u00fa\u0001\u0000\u0000"+ - "\u0000\u03ee\u03ef\u0003\u00f7v\u0000\u03ef\u03f0\u0001\u0000\u0000\u0000"+ - "\u03f0\u03f1\u0006x\u0015\u0000\u03f1\u00fc\u0001\u0000\u0000\u0000\u03f2"+ - "\u03f3\u0003/\u0012\u0000\u03f3\u03f4\u0001\u0000\u0000\u0000\u03f4\u03f5"+ - "\u0006y\b\u0000\u03f5\u00fe\u0001\u0000\u0000\u0000\u03f6\u03f7\u0003"+ - "1\u0013\u0000\u03f7\u03f8\u0001\u0000\u0000\u0000\u03f8\u03f9\u0006z\b"+ - "\u0000\u03f9\u0100\u0001\u0000\u0000\u0000\u03fa\u03fb\u00033\u0014\u0000"+ - "\u03fb\u03fc\u0001\u0000\u0000\u0000\u03fc\u03fd\u0006{\b\u0000\u03fd"+ - "\u0102\u0001\u0000\u0000\u0000\u03fe\u03ff\u0003?\u001a\u0000\u03ff\u0400"+ - "\u0001\u0000\u0000\u0000\u0400\u0401\u0006|\u000b\u0000\u0401\u0402\u0006"+ - "|\f\u0000\u0402\u0403\u0006|\f\u0000\u0403\u0104\u0001\u0000\u0000\u0000"+ - "\u0404\u0405\u0003a+\u0000\u0405\u0406\u0001\u0000\u0000\u0000\u0406\u0407"+ - "\u0006}\u000f\u0000\u0407\u0106\u0001\u0000\u0000\u0000\u0408\u0409\u0003"+ - "c,\u0000\u0409\u040a\u0001\u0000\u0000\u0000\u040a\u040b\u0006~\u000e"+ - "\u0000\u040b\u0108\u0001\u0000\u0000\u0000\u040c\u040d\u0003g.\u0000\u040d"+ - "\u040e\u0001\u0000\u0000\u0000\u040e\u040f\u0006\u007f\u0011\u0000\u040f"+ - "\u010a\u0001\u0000\u0000\u0000\u0410\u0411\u0003\u00f3t\u0000\u0411\u0412"+ - "\u0001\u0000\u0000\u0000\u0412\u0413\u0006\u0080\u0016\u0000\u0413\u010c"+ - "\u0001\u0000\u0000\u0000\u0414\u0415\u0003\u00cda\u0000\u0415\u0416\u0001"+ - "\u0000\u0000\u0000\u0416\u0417\u0006\u0081\u0012\u0000\u0417\u010e\u0001"+ - "\u0000\u0000\u0000\u0418\u0419\u0003\u00a5M\u0000\u0419\u041a\u0001\u0000"+ - "\u0000\u0000\u041a\u041b\u0006\u0082\u0010\u0000\u041b\u0110\u0001\u0000"+ - "\u0000\u0000\u041c\u041d\u0003/\u0012\u0000\u041d\u041e\u0001\u0000\u0000"+ - "\u0000\u041e\u041f\u0006\u0083\b\u0000\u041f\u0112\u0001\u0000\u0000\u0000"+ - "\u0420\u0421\u00031\u0013\u0000\u0421\u0422\u0001\u0000\u0000\u0000\u0422"+ - "\u0423\u0006\u0084\b\u0000\u0423\u0114\u0001\u0000\u0000\u0000\u0424\u0425"+ - "\u00033\u0014\u0000\u0425\u0426\u0001\u0000\u0000\u0000\u0426\u0427\u0006"+ - "\u0085\b\u0000\u0427\u0116\u0001\u0000\u0000\u0000\u0428\u0429\u0003?"+ - "\u001a\u0000\u0429\u042a\u0001\u0000\u0000\u0000\u042a\u042b\u0006\u0086"+ - "\u000b\u0000\u042b\u042c\u0006\u0086\f\u0000\u042c\u0118\u0001\u0000\u0000"+ - "\u0000\u042d\u042e\u0003g.\u0000\u042e\u042f\u0001\u0000\u0000\u0000\u042f"+ - "\u0430\u0006\u0087\u0011\u0000\u0430\u011a\u0001\u0000\u0000\u0000\u0431"+ - "\u0432\u0003\u00a5M\u0000\u0432\u0433\u0001\u0000\u0000\u0000\u0433\u0434"+ - "\u0006\u0088\u0010\u0000\u0434\u011c\u0001\u0000\u0000\u0000\u0435\u0436"+ - "\u0003\u00a3L\u0000\u0436\u0437\u0001\u0000\u0000\u0000\u0437\u0438\u0006"+ - "\u0089\u0017\u0000\u0438\u011e\u0001\u0000\u0000\u0000\u0439\u043a\u0003"+ - "/\u0012\u0000\u043a\u043b\u0001\u0000\u0000\u0000\u043b\u043c\u0006\u008a"+ - "\b\u0000\u043c\u0120\u0001\u0000\u0000\u0000\u043d\u043e\u00031\u0013"+ - "\u0000\u043e\u043f\u0001\u0000\u0000\u0000\u043f\u0440\u0006\u008b\b\u0000"+ - "\u0440\u0122\u0001\u0000\u0000\u0000\u0441\u0442\u00033\u0014\u0000\u0442"+ - "\u0443\u0001\u0000\u0000\u0000\u0443\u0444\u0006\u008c\b\u0000\u0444\u0124"+ - "\u0001\u0000\u0000\u0000\u0445\u0446\u0003?\u001a\u0000\u0446\u0447\u0001"+ - "\u0000\u0000\u0000\u0447\u0448\u0006\u008d\u000b\u0000\u0448\u0449\u0006"+ - "\u008d\f\u0000\u0449\u0126\u0001\u0000\u0000\u0000\u044a\u044b\u0005i"+ - "\u0000\u0000\u044b\u044c\u0005n\u0000\u0000\u044c\u044d\u0005f\u0000\u0000"+ - "\u044d\u044e\u0005o\u0000\u0000\u044e\u0128\u0001\u0000\u0000\u0000\u044f"+ - "\u0450\u0005f\u0000\u0000\u0450\u0451\u0005u\u0000\u0000\u0451\u0452\u0005"+ - "n\u0000\u0000\u0452\u0453\u0005c\u0000\u0000\u0453\u0454\u0005t\u0000"+ - "\u0000\u0454\u0455\u0005i\u0000\u0000\u0455\u0456\u0005o\u0000\u0000\u0456"+ - "\u0457\u0005n\u0000\u0000\u0457\u0458\u0005s\u0000\u0000\u0458\u012a\u0001"+ - "\u0000\u0000\u0000\u0459\u045a\u0003/\u0012\u0000\u045a\u045b\u0001\u0000"+ - "\u0000\u0000\u045b\u045c\u0006\u0090\b\u0000\u045c\u012c\u0001\u0000\u0000"+ - "\u0000\u045d\u045e\u00031\u0013\u0000\u045e\u045f\u0001\u0000\u0000\u0000"+ - "\u045f\u0460\u0006\u0091\b\u0000\u0460\u012e\u0001\u0000\u0000\u0000\u0461"+ - "\u0462\u00033\u0014\u0000\u0462\u0463\u0001\u0000\u0000\u0000\u0463\u0464"+ - "\u0006\u0092\b\u0000\u0464\u0130\u0001\u0000\u0000\u0000\u0465\u0466\u0003"+ - "\u00a1K\u0000\u0466\u0467\u0001\u0000\u0000\u0000\u0467\u0468\u0006\u0093"+ - "\r\u0000\u0468\u0469\u0006\u0093\f\u0000\u0469\u0132\u0001\u0000\u0000"+ - "\u0000\u046a\u046b\u0005:\u0000\u0000\u046b\u0134\u0001\u0000\u0000\u0000"+ - "\u046c\u0472\u0003K \u0000\u046d\u0472\u0003A\u001b\u0000\u046e\u0472"+ - "\u0003g.\u0000\u046f\u0472\u0003C\u001c\u0000\u0470\u0472\u0003Q#\u0000"+ - "\u0471\u046c\u0001\u0000\u0000\u0000\u0471\u046d\u0001\u0000\u0000\u0000"+ - "\u0471\u046e\u0001\u0000\u0000\u0000\u0471\u046f\u0001\u0000\u0000\u0000"+ - "\u0471\u0470\u0001\u0000\u0000\u0000\u0472\u0473\u0001\u0000\u0000\u0000"+ - "\u0473\u0471\u0001\u0000\u0000\u0000\u0473\u0474\u0001\u0000\u0000\u0000"+ - "\u0474\u0136\u0001\u0000\u0000\u0000\u0475\u0476\u0003/\u0012\u0000\u0476"+ - "\u0477\u0001\u0000\u0000\u0000\u0477\u0478\u0006\u0096\b\u0000\u0478\u0138"+ - "\u0001\u0000\u0000\u0000\u0479\u047a\u00031\u0013\u0000\u047a\u047b\u0001"+ - "\u0000\u0000\u0000\u047b\u047c\u0006\u0097\b\u0000\u047c\u013a\u0001\u0000"+ - "\u0000\u0000\u047d\u047e\u00033\u0014\u0000\u047e\u047f\u0001\u0000\u0000"+ - "\u0000\u047f\u0480\u0006\u0098\b\u0000\u0480\u013c\u0001\u0000\u0000\u0000"+ - "7\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u01cf\u01d9\u01dd"+ - "\u01e0\u01e9\u01eb\u01f6\u021f\u0224\u022d\u0234\u0239\u023b\u0246\u024e"+ - "\u0251\u0253\u0258\u025d\u0263\u026a\u026f\u0275\u0278\u0280\u0284\u0306"+ - "\u030b\u0310\u0312\u0318\u0349\u034e\u0371\u0375\u037a\u037f\u0384\u0386"+ - "\u03df\u03e3\u03e8\u0471\u0473\u0018\u0005\u0002\u0000\u0005\u0004\u0000"+ - "\u0005\u0006\u0000\u0005\u0001\u0000\u0005\u0003\u0000\u0005\b\u0000\u0005"+ - "\u0005\u0000\u0005\t\u0000\u0000\u0001\u0000\u0007?\u0000\u0005\u0000"+ - "\u0000\u0007\u0019\u0000\u0004\u0000\u0000\u0007@\u0000\u0007!\u0000\u0007"+ - " \u0000\u0007B\u0000\u0007#\u0000\u0007K\u0000\u0005\n\u0000\u0005\u0007"+ - "\u0000\u0007U\u0000\u0007T\u0000\u0007A\u0000"; + "\u0386\u037f\u0001\u0000\u0000\u0000\u0387\u00d0\u0001\u0000\u0000\u0000"+ + "\u0388\u038b\u0003\u00cfb\u0000\u0389\u038b\u0003\u00a5M\u0000\u038a\u0388"+ + "\u0001\u0000\u0000\u0000\u038a\u0389\u0001\u0000\u0000\u0000\u038b\u038c"+ + "\u0001\u0000\u0000\u0000\u038c\u038a\u0001\u0000\u0000\u0000\u038c\u038d"+ + "\u0001\u0000\u0000\u0000\u038d\u00d2\u0001\u0000\u0000\u0000\u038e\u038f"+ + "\u0003/\u0012\u0000\u038f\u0390\u0001\u0000\u0000\u0000\u0390\u0391\u0006"+ + "d\b\u0000\u0391\u00d4\u0001\u0000\u0000\u0000\u0392\u0393\u00031\u0013"+ + "\u0000\u0393\u0394\u0001\u0000\u0000\u0000\u0394\u0395\u0006e\b\u0000"+ + "\u0395\u00d6\u0001\u0000\u0000\u0000\u0396\u0397\u00033\u0014\u0000\u0397"+ + "\u0398\u0001\u0000\u0000\u0000\u0398\u0399\u0006f\b\u0000\u0399\u00d8"+ + "\u0001\u0000\u0000\u0000\u039a\u039b\u0003?\u001a\u0000\u039b\u039c\u0001"+ + "\u0000\u0000\u0000\u039c\u039d\u0006g\u000b\u0000\u039d\u039e\u0006g\f"+ + "\u0000\u039e\u00da\u0001\u0000\u0000\u0000\u039f\u03a0\u0003a+\u0000\u03a0"+ + "\u03a1\u0001\u0000\u0000\u0000\u03a1\u03a2\u0006h\u000f\u0000\u03a2\u00dc"+ + "\u0001\u0000\u0000\u0000\u03a3\u03a4\u0003c,\u0000\u03a4\u03a5\u0001\u0000"+ + "\u0000\u0000\u03a5\u03a6\u0006i\u000e\u0000\u03a6\u00de\u0001\u0000\u0000"+ + "\u0000\u03a7\u03a8\u0003g.\u0000\u03a8\u03a9\u0001\u0000\u0000\u0000\u03a9"+ + "\u03aa\u0006j\u0011\u0000\u03aa\u00e0\u0001\u0000\u0000\u0000\u03ab\u03ac"+ + "\u0005a\u0000\u0000\u03ac\u03ad\u0005s\u0000\u0000\u03ad\u00e2\u0001\u0000"+ + "\u0000\u0000\u03ae\u03af\u0003\u00d1c\u0000\u03af\u03b0\u0001\u0000\u0000"+ + "\u0000\u03b0\u03b1\u0006l\u0012\u0000\u03b1\u00e4\u0001\u0000\u0000\u0000"+ + "\u03b2\u03b3\u0003/\u0012\u0000\u03b3\u03b4\u0001\u0000\u0000\u0000\u03b4"+ + "\u03b5\u0006m\b\u0000\u03b5\u00e6\u0001\u0000\u0000\u0000\u03b6\u03b7"+ + "\u00031\u0013\u0000\u03b7\u03b8\u0001\u0000\u0000\u0000\u03b8\u03b9\u0006"+ + "n\b\u0000\u03b9\u00e8\u0001\u0000\u0000\u0000\u03ba\u03bb\u00033\u0014"+ + "\u0000\u03bb\u03bc\u0001\u0000\u0000\u0000\u03bc\u03bd\u0006o\b\u0000"+ + "\u03bd\u00ea\u0001\u0000\u0000\u0000\u03be\u03bf\u0003?\u001a\u0000\u03bf"+ + "\u03c0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006p\u000b\u0000\u03c1\u03c2"+ + "\u0006p\f\u0000\u03c2\u00ec\u0001\u0000\u0000\u0000\u03c3\u03c4\u0003"+ + "\u009fJ\u0000\u03c4\u03c5\u0001\u0000\u0000\u0000\u03c5\u03c6\u0006q\t"+ + "\u0000\u03c6\u03c7\u0006q\u0013\u0000\u03c7\u00ee\u0001\u0000\u0000\u0000"+ + "\u03c8\u03c9\u0005o\u0000\u0000\u03c9\u03ca\u0005n\u0000\u0000\u03ca\u03cb"+ + "\u0001\u0000\u0000\u0000\u03cb\u03cc\u0006r\u0014\u0000\u03cc\u00f0\u0001"+ + "\u0000\u0000\u0000\u03cd\u03ce\u0005w\u0000\u0000\u03ce\u03cf\u0005i\u0000"+ + "\u0000\u03cf\u03d0\u0005t\u0000\u0000\u03d0\u03d1\u0005h\u0000\u0000\u03d1"+ + "\u03d2\u0001\u0000\u0000\u0000\u03d2\u03d3\u0006s\u0014\u0000\u03d3\u00f2"+ + "\u0001\u0000\u0000\u0000\u03d4\u03d5\b\f\u0000\u0000\u03d5\u00f4\u0001"+ + "\u0000\u0000\u0000\u03d6\u03d8\u0003\u00f3t\u0000\u03d7\u03d6\u0001\u0000"+ + "\u0000\u0000\u03d8\u03d9\u0001\u0000\u0000\u0000\u03d9\u03d7\u0001\u0000"+ + "\u0000\u0000\u03d9\u03da\u0001\u0000\u0000\u0000\u03da\u03db\u0001\u0000"+ + "\u0000\u0000\u03db\u03dc\u0003\u0131\u0093\u0000\u03dc\u03de\u0001\u0000"+ + "\u0000\u0000\u03dd\u03d7\u0001\u0000\u0000\u0000\u03dd\u03de\u0001\u0000"+ + "\u0000\u0000\u03de\u03e0\u0001\u0000\u0000\u0000\u03df\u03e1\u0003\u00f3"+ + "t\u0000\u03e0\u03df\u0001\u0000\u0000\u0000\u03e1\u03e2\u0001\u0000\u0000"+ + "\u0000\u03e2\u03e0\u0001\u0000\u0000\u0000\u03e2\u03e3\u0001\u0000\u0000"+ + "\u0000\u03e3\u00f6\u0001\u0000\u0000\u0000\u03e4\u03e5\u0003\u00a7N\u0000"+ + "\u03e5\u03e6\u0001\u0000\u0000\u0000\u03e6\u03e7\u0006v\u0010\u0000\u03e7"+ + "\u00f8\u0001\u0000\u0000\u0000\u03e8\u03e9\u0003\u00f5u\u0000\u03e9\u03ea"+ + "\u0001\u0000\u0000\u0000\u03ea\u03eb\u0006w\u0015\u0000\u03eb\u00fa\u0001"+ + "\u0000\u0000\u0000\u03ec\u03ed\u0003/\u0012\u0000\u03ed\u03ee\u0001\u0000"+ + "\u0000\u0000\u03ee\u03ef\u0006x\b\u0000\u03ef\u00fc\u0001\u0000\u0000"+ + "\u0000\u03f0\u03f1\u00031\u0013\u0000\u03f1\u03f2\u0001\u0000\u0000\u0000"+ + "\u03f2\u03f3\u0006y\b\u0000\u03f3\u00fe\u0001\u0000\u0000\u0000\u03f4"+ + "\u03f5\u00033\u0014\u0000\u03f5\u03f6\u0001\u0000\u0000\u0000\u03f6\u03f7"+ + "\u0006z\b\u0000\u03f7\u0100\u0001\u0000\u0000\u0000\u03f8\u03f9\u0003"+ + "?\u001a\u0000\u03f9\u03fa\u0001\u0000\u0000\u0000\u03fa\u03fb\u0006{\u000b"+ + "\u0000\u03fb\u03fc\u0006{\f\u0000\u03fc\u03fd\u0006{\f\u0000\u03fd\u0102"+ + "\u0001\u0000\u0000\u0000\u03fe\u03ff\u0003a+\u0000\u03ff\u0400\u0001\u0000"+ + "\u0000\u0000\u0400\u0401\u0006|\u000f\u0000\u0401\u0104\u0001\u0000\u0000"+ + "\u0000\u0402\u0403\u0003c,\u0000\u0403\u0404\u0001\u0000\u0000\u0000\u0404"+ + "\u0405\u0006}\u000e\u0000\u0405\u0106\u0001\u0000\u0000\u0000\u0406\u0407"+ + "\u0003g.\u0000\u0407\u0408\u0001\u0000\u0000\u0000\u0408\u0409\u0006~"+ + "\u0011\u0000\u0409\u0108\u0001\u0000\u0000\u0000\u040a\u040b\u0003\u00f1"+ + "s\u0000\u040b\u040c\u0001\u0000\u0000\u0000\u040c\u040d\u0006\u007f\u0016"+ + "\u0000\u040d\u010a\u0001\u0000\u0000\u0000\u040e\u040f\u0003\u00d1c\u0000"+ + "\u040f\u0410\u0001\u0000\u0000\u0000\u0410\u0411\u0006\u0080\u0012\u0000"+ + "\u0411\u010c\u0001\u0000\u0000\u0000\u0412\u0413\u0003\u00a7N\u0000\u0413"+ + "\u0414\u0001\u0000\u0000\u0000\u0414\u0415\u0006\u0081\u0010\u0000\u0415"+ + "\u010e\u0001\u0000\u0000\u0000\u0416\u0417\u0003/\u0012\u0000\u0417\u0418"+ + "\u0001\u0000\u0000\u0000\u0418\u0419\u0006\u0082\b\u0000\u0419\u0110\u0001"+ + "\u0000\u0000\u0000\u041a\u041b\u00031\u0013\u0000\u041b\u041c\u0001\u0000"+ + "\u0000\u0000\u041c\u041d\u0006\u0083\b\u0000\u041d\u0112\u0001\u0000\u0000"+ + "\u0000\u041e\u041f\u00033\u0014\u0000\u041f\u0420\u0001\u0000\u0000\u0000"+ + "\u0420\u0421\u0006\u0084\b\u0000\u0421\u0114\u0001\u0000\u0000\u0000\u0422"+ + "\u0423\u0003?\u001a\u0000\u0423\u0424\u0001\u0000\u0000\u0000\u0424\u0425"+ + "\u0006\u0085\u000b\u0000\u0425\u0426\u0006\u0085\f\u0000\u0426\u0116\u0001"+ + "\u0000\u0000\u0000\u0427\u0428\u0003g.\u0000\u0428\u0429\u0001\u0000\u0000"+ + "\u0000\u0429\u042a\u0006\u0086\u0011\u0000\u042a\u0118\u0001\u0000\u0000"+ + "\u0000\u042b\u042c\u0003\u00a7N\u0000\u042c\u042d\u0001\u0000\u0000\u0000"+ + "\u042d\u042e\u0006\u0087\u0010\u0000\u042e\u011a\u0001\u0000\u0000\u0000"+ + "\u042f\u0430\u0003\u00a3L\u0000\u0430\u0431\u0001\u0000\u0000\u0000\u0431"+ + "\u0432\u0006\u0088\u0017\u0000\u0432\u011c\u0001\u0000\u0000\u0000\u0433"+ + "\u0434\u0003/\u0012\u0000\u0434\u0435\u0001\u0000\u0000\u0000\u0435\u0436"+ + "\u0006\u0089\b\u0000\u0436\u011e\u0001\u0000\u0000\u0000\u0437\u0438\u0003"+ + "1\u0013\u0000\u0438\u0439\u0001\u0000\u0000\u0000\u0439\u043a\u0006\u008a"+ + "\b\u0000\u043a\u0120\u0001\u0000\u0000\u0000\u043b\u043c\u00033\u0014"+ + "\u0000\u043c\u043d\u0001\u0000\u0000\u0000\u043d\u043e\u0006\u008b\b\u0000"+ + "\u043e\u0122\u0001\u0000\u0000\u0000\u043f\u0440\u0003?\u001a\u0000\u0440"+ + "\u0441\u0001\u0000\u0000\u0000\u0441\u0442\u0006\u008c\u000b\u0000\u0442"+ + "\u0443\u0006\u008c\f\u0000\u0443\u0124\u0001\u0000\u0000\u0000\u0444\u0445"+ + "\u0005i\u0000\u0000\u0445\u0446\u0005n\u0000\u0000\u0446\u0447\u0005f"+ + "\u0000\u0000\u0447\u0448\u0005o\u0000\u0000\u0448\u0126\u0001\u0000\u0000"+ + "\u0000\u0449\u044a\u0005f\u0000\u0000\u044a\u044b\u0005u\u0000\u0000\u044b"+ + "\u044c\u0005n\u0000\u0000\u044c\u044d\u0005c\u0000\u0000\u044d\u044e\u0005"+ + "t\u0000\u0000\u044e\u044f\u0005i\u0000\u0000\u044f\u0450\u0005o\u0000"+ + "\u0000\u0450\u0451\u0005n\u0000\u0000\u0451\u0452\u0005s\u0000\u0000\u0452"+ + "\u0128\u0001\u0000\u0000\u0000\u0453\u0454\u0003/\u0012\u0000\u0454\u0455"+ + "\u0001\u0000\u0000\u0000\u0455\u0456\u0006\u008f\b\u0000\u0456\u012a\u0001"+ + "\u0000\u0000\u0000\u0457\u0458\u00031\u0013\u0000\u0458\u0459\u0001\u0000"+ + "\u0000\u0000\u0459\u045a\u0006\u0090\b\u0000\u045a\u012c\u0001\u0000\u0000"+ + "\u0000\u045b\u045c\u00033\u0014\u0000\u045c\u045d\u0001\u0000\u0000\u0000"+ + "\u045d\u045e\u0006\u0091\b\u0000\u045e\u012e\u0001\u0000\u0000\u0000\u045f"+ + "\u0460\u0003\u00a1K\u0000\u0460\u0461\u0001\u0000\u0000\u0000\u0461\u0462"+ + "\u0006\u0092\r\u0000\u0462\u0463\u0006\u0092\f\u0000\u0463\u0130\u0001"+ + "\u0000\u0000\u0000\u0464\u0465\u0005:\u0000\u0000\u0465\u0132\u0001\u0000"+ + "\u0000\u0000\u0466\u046c\u0003K \u0000\u0467\u046c\u0003A\u001b\u0000"+ + "\u0468\u046c\u0003g.\u0000\u0469\u046c\u0003C\u001c\u0000\u046a\u046c"+ + "\u0003Q#\u0000\u046b\u0466\u0001\u0000\u0000\u0000\u046b\u0467\u0001\u0000"+ + "\u0000\u0000\u046b\u0468\u0001\u0000\u0000\u0000\u046b\u0469\u0001\u0000"+ + "\u0000\u0000\u046b\u046a\u0001\u0000\u0000\u0000\u046c\u046d\u0001\u0000"+ + "\u0000\u0000\u046d\u046b\u0001\u0000\u0000\u0000\u046d\u046e\u0001\u0000"+ + "\u0000\u0000\u046e\u0134\u0001\u0000\u0000\u0000\u046f\u0470\u0003/\u0012"+ + "\u0000\u0470\u0471\u0001\u0000\u0000\u0000\u0471\u0472\u0006\u0095\b\u0000"+ + "\u0472\u0136\u0001\u0000\u0000\u0000\u0473\u0474\u00031\u0013\u0000\u0474"+ + "\u0475\u0001\u0000\u0000\u0000\u0475\u0476\u0006\u0096\b\u0000\u0476\u0138"+ + "\u0001\u0000\u0000\u0000\u0477\u0478\u00033\u0014\u0000\u0478\u0479\u0001"+ + "\u0000\u0000\u0000\u0479\u047a\u0006\u0097\b\u0000\u047a\u013a\u0001\u0000"+ + "\u0000\u00009\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u01cd"+ + "\u01d7\u01db\u01de\u01e7\u01e9\u01f4\u021d\u0222\u022b\u0232\u0237\u0239"+ + "\u0244\u024c\u024f\u0251\u0256\u025b\u0261\u0268\u026d\u0273\u0276\u027e"+ + "\u0282\u0304\u0309\u030e\u0310\u0316\u0349\u034e\u0371\u0375\u037a\u037f"+ + "\u0384\u0386\u038a\u038c\u03d9\u03dd\u03e2\u046b\u046d\u0018\u0005\u0002"+ + "\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0001\u0000\u0005\u0003"+ + "\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0000\u0001\u0000"+ + "\u0007?\u0000\u0005\u0000\u0000\u0007\u0019\u0000\u0004\u0000\u0000\u0007"+ + "@\u0000\u0007!\u0000\u0007 \u0000\u0007B\u0000\u0007#\u0000\u0007K\u0000"+ + "\u0005\n\u0000\u0005\u0007\u0000\u0007U\u0000\u0007T\u0000\u0007A\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index 7d5b63a95b667..f4348a84eff69 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -181,7 +181,7 @@ FROM_UNQUOTED_IDENTIFIER FROM_LINE_COMMENT FROM_MULTILINE_COMMENT FROM_WS -UNQUOTED_ID_PATTERN +ID_PATTERN PROJECT_LINE_COMMENT PROJECT_MULTILINE_COMMENT PROJECT_WS @@ -239,7 +239,6 @@ qualifiedName qualifiedNamePattern identifier identifierPattern -idPattern constant limitCommand sortCommand @@ -267,4 +266,4 @@ enrichWithClause atn: -[4, 1, 104, 514, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 112, 8, 1, 10, 1, 12, 1, 115, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 121, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 136, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 148, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 155, 8, 5, 10, 5, 12, 5, 158, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 165, 8, 5, 1, 5, 1, 5, 3, 5, 169, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 177, 8, 5, 10, 5, 12, 5, 180, 9, 5, 1, 6, 1, 6, 3, 6, 184, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 191, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 196, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 203, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 209, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 217, 8, 8, 10, 8, 12, 8, 220, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 229, 8, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 237, 8, 10, 10, 10, 12, 10, 240, 9, 10, 3, 10, 242, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 5, 12, 252, 8, 12, 10, 12, 12, 12, 255, 9, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 262, 8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 5, 14, 268, 8, 14, 10, 14, 12, 14, 271, 9, 14, 1, 14, 3, 14, 274, 8, 14, 1, 15, 1, 15, 3, 15, 278, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 284, 8, 16, 10, 16, 12, 16, 287, 9, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 3, 19, 298, 8, 19, 1, 19, 1, 19, 3, 19, 302, 8, 19, 1, 20, 1, 20, 1, 20, 1, 20, 3, 20, 308, 8, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 5, 22, 315, 8, 22, 10, 22, 12, 22, 318, 9, 22, 1, 23, 1, 23, 1, 23, 5, 23, 323, 8, 23, 10, 23, 12, 23, 326, 9, 23, 1, 24, 1, 24, 1, 25, 4, 25, 331, 8, 25, 11, 25, 12, 25, 332, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 350, 8, 27, 10, 27, 12, 27, 353, 9, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 361, 8, 27, 10, 27, 12, 27, 364, 9, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 372, 8, 27, 10, 27, 12, 27, 375, 9, 27, 1, 27, 1, 27, 3, 27, 379, 8, 27, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 388, 8, 29, 10, 29, 12, 29, 391, 9, 29, 1, 30, 1, 30, 3, 30, 395, 8, 30, 1, 30, 1, 30, 3, 30, 399, 8, 30, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 405, 8, 31, 10, 31, 12, 31, 408, 9, 31, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 414, 8, 32, 10, 32, 12, 32, 417, 9, 32, 1, 33, 1, 33, 1, 33, 1, 33, 5, 33, 423, 8, 33, 10, 33, 12, 33, 426, 9, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 35, 3, 35, 436, 8, 35, 1, 36, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 5, 38, 448, 8, 38, 10, 38, 12, 38, 451, 9, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 3, 41, 461, 8, 41, 1, 42, 3, 42, 464, 8, 42, 1, 42, 1, 42, 1, 43, 3, 43, 469, 8, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 3, 48, 488, 8, 48, 1, 49, 1, 49, 1, 49, 1, 49, 3, 49, 494, 8, 49, 1, 49, 1, 49, 1, 49, 1, 49, 5, 49, 500, 8, 49, 10, 49, 12, 49, 503, 9, 49, 3, 49, 505, 8, 49, 1, 50, 1, 50, 1, 50, 3, 50, 510, 8, 50, 1, 50, 1, 50, 1, 50, 0, 3, 2, 10, 16, 51, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 0, 9, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 66, 66, 71, 71, 1, 0, 65, 66, 2, 0, 66, 66, 75, 75, 2, 0, 31, 31, 34, 34, 1, 0, 37, 38, 2, 0, 36, 36, 50, 50, 1, 0, 51, 57, 540, 0, 102, 1, 0, 0, 0, 2, 105, 1, 0, 0, 0, 4, 120, 1, 0, 0, 0, 6, 135, 1, 0, 0, 0, 8, 137, 1, 0, 0, 0, 10, 168, 1, 0, 0, 0, 12, 195, 1, 0, 0, 0, 14, 202, 1, 0, 0, 0, 16, 208, 1, 0, 0, 0, 18, 228, 1, 0, 0, 0, 20, 230, 1, 0, 0, 0, 22, 245, 1, 0, 0, 0, 24, 248, 1, 0, 0, 0, 26, 261, 1, 0, 0, 0, 28, 263, 1, 0, 0, 0, 30, 277, 1, 0, 0, 0, 32, 279, 1, 0, 0, 0, 34, 288, 1, 0, 0, 0, 36, 292, 1, 0, 0, 0, 38, 295, 1, 0, 0, 0, 40, 303, 1, 0, 0, 0, 42, 309, 1, 0, 0, 0, 44, 311, 1, 0, 0, 0, 46, 319, 1, 0, 0, 0, 48, 327, 1, 0, 0, 0, 50, 330, 1, 0, 0, 0, 52, 334, 1, 0, 0, 0, 54, 378, 1, 0, 0, 0, 56, 380, 1, 0, 0, 0, 58, 383, 1, 0, 0, 0, 60, 392, 1, 0, 0, 0, 62, 400, 1, 0, 0, 0, 64, 409, 1, 0, 0, 0, 66, 418, 1, 0, 0, 0, 68, 427, 1, 0, 0, 0, 70, 431, 1, 0, 0, 0, 72, 437, 1, 0, 0, 0, 74, 441, 1, 0, 0, 0, 76, 444, 1, 0, 0, 0, 78, 452, 1, 0, 0, 0, 80, 456, 1, 0, 0, 0, 82, 460, 1, 0, 0, 0, 84, 463, 1, 0, 0, 0, 86, 468, 1, 0, 0, 0, 88, 472, 1, 0, 0, 0, 90, 474, 1, 0, 0, 0, 92, 476, 1, 0, 0, 0, 94, 479, 1, 0, 0, 0, 96, 487, 1, 0, 0, 0, 98, 489, 1, 0, 0, 0, 100, 509, 1, 0, 0, 0, 102, 103, 3, 2, 1, 0, 103, 104, 5, 0, 0, 1, 104, 1, 1, 0, 0, 0, 105, 106, 6, 1, -1, 0, 106, 107, 3, 4, 2, 0, 107, 113, 1, 0, 0, 0, 108, 109, 10, 1, 0, 0, 109, 110, 5, 25, 0, 0, 110, 112, 3, 6, 3, 0, 111, 108, 1, 0, 0, 0, 112, 115, 1, 0, 0, 0, 113, 111, 1, 0, 0, 0, 113, 114, 1, 0, 0, 0, 114, 3, 1, 0, 0, 0, 115, 113, 1, 0, 0, 0, 116, 121, 3, 92, 46, 0, 117, 121, 3, 28, 14, 0, 118, 121, 3, 22, 11, 0, 119, 121, 3, 96, 48, 0, 120, 116, 1, 0, 0, 0, 120, 117, 1, 0, 0, 0, 120, 118, 1, 0, 0, 0, 120, 119, 1, 0, 0, 0, 121, 5, 1, 0, 0, 0, 122, 136, 3, 36, 18, 0, 123, 136, 3, 40, 20, 0, 124, 136, 3, 56, 28, 0, 125, 136, 3, 62, 31, 0, 126, 136, 3, 58, 29, 0, 127, 136, 3, 38, 19, 0, 128, 136, 3, 8, 4, 0, 129, 136, 3, 64, 32, 0, 130, 136, 3, 66, 33, 0, 131, 136, 3, 70, 35, 0, 132, 136, 3, 72, 36, 0, 133, 136, 3, 98, 49, 0, 134, 136, 3, 74, 37, 0, 135, 122, 1, 0, 0, 0, 135, 123, 1, 0, 0, 0, 135, 124, 1, 0, 0, 0, 135, 125, 1, 0, 0, 0, 135, 126, 1, 0, 0, 0, 135, 127, 1, 0, 0, 0, 135, 128, 1, 0, 0, 0, 135, 129, 1, 0, 0, 0, 135, 130, 1, 0, 0, 0, 135, 131, 1, 0, 0, 0, 135, 132, 1, 0, 0, 0, 135, 133, 1, 0, 0, 0, 135, 134, 1, 0, 0, 0, 136, 7, 1, 0, 0, 0, 137, 138, 5, 17, 0, 0, 138, 139, 3, 10, 5, 0, 139, 9, 1, 0, 0, 0, 140, 141, 6, 5, -1, 0, 141, 142, 5, 43, 0, 0, 142, 169, 3, 10, 5, 7, 143, 169, 3, 14, 7, 0, 144, 169, 3, 12, 6, 0, 145, 147, 3, 14, 7, 0, 146, 148, 5, 43, 0, 0, 147, 146, 1, 0, 0, 0, 147, 148, 1, 0, 0, 0, 148, 149, 1, 0, 0, 0, 149, 150, 5, 40, 0, 0, 150, 151, 5, 39, 0, 0, 151, 156, 3, 14, 7, 0, 152, 153, 5, 33, 0, 0, 153, 155, 3, 14, 7, 0, 154, 152, 1, 0, 0, 0, 155, 158, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 156, 157, 1, 0, 0, 0, 157, 159, 1, 0, 0, 0, 158, 156, 1, 0, 0, 0, 159, 160, 5, 49, 0, 0, 160, 169, 1, 0, 0, 0, 161, 162, 3, 14, 7, 0, 162, 164, 5, 41, 0, 0, 163, 165, 5, 43, 0, 0, 164, 163, 1, 0, 0, 0, 164, 165, 1, 0, 0, 0, 165, 166, 1, 0, 0, 0, 166, 167, 5, 44, 0, 0, 167, 169, 1, 0, 0, 0, 168, 140, 1, 0, 0, 0, 168, 143, 1, 0, 0, 0, 168, 144, 1, 0, 0, 0, 168, 145, 1, 0, 0, 0, 168, 161, 1, 0, 0, 0, 169, 178, 1, 0, 0, 0, 170, 171, 10, 4, 0, 0, 171, 172, 5, 30, 0, 0, 172, 177, 3, 10, 5, 5, 173, 174, 10, 3, 0, 0, 174, 175, 5, 46, 0, 0, 175, 177, 3, 10, 5, 4, 176, 170, 1, 0, 0, 0, 176, 173, 1, 0, 0, 0, 177, 180, 1, 0, 0, 0, 178, 176, 1, 0, 0, 0, 178, 179, 1, 0, 0, 0, 179, 11, 1, 0, 0, 0, 180, 178, 1, 0, 0, 0, 181, 183, 3, 14, 7, 0, 182, 184, 5, 43, 0, 0, 183, 182, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 5, 42, 0, 0, 186, 187, 3, 88, 44, 0, 187, 196, 1, 0, 0, 0, 188, 190, 3, 14, 7, 0, 189, 191, 5, 43, 0, 0, 190, 189, 1, 0, 0, 0, 190, 191, 1, 0, 0, 0, 191, 192, 1, 0, 0, 0, 192, 193, 5, 48, 0, 0, 193, 194, 3, 88, 44, 0, 194, 196, 1, 0, 0, 0, 195, 181, 1, 0, 0, 0, 195, 188, 1, 0, 0, 0, 196, 13, 1, 0, 0, 0, 197, 203, 3, 16, 8, 0, 198, 199, 3, 16, 8, 0, 199, 200, 3, 90, 45, 0, 200, 201, 3, 16, 8, 0, 201, 203, 1, 0, 0, 0, 202, 197, 1, 0, 0, 0, 202, 198, 1, 0, 0, 0, 203, 15, 1, 0, 0, 0, 204, 205, 6, 8, -1, 0, 205, 209, 3, 18, 9, 0, 206, 207, 7, 0, 0, 0, 207, 209, 3, 16, 8, 3, 208, 204, 1, 0, 0, 0, 208, 206, 1, 0, 0, 0, 209, 218, 1, 0, 0, 0, 210, 211, 10, 2, 0, 0, 211, 212, 7, 1, 0, 0, 212, 217, 3, 16, 8, 3, 213, 214, 10, 1, 0, 0, 214, 215, 7, 0, 0, 0, 215, 217, 3, 16, 8, 2, 216, 210, 1, 0, 0, 0, 216, 213, 1, 0, 0, 0, 217, 220, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219, 17, 1, 0, 0, 0, 220, 218, 1, 0, 0, 0, 221, 229, 3, 54, 27, 0, 222, 229, 3, 44, 22, 0, 223, 229, 3, 20, 10, 0, 224, 225, 5, 39, 0, 0, 225, 226, 3, 10, 5, 0, 226, 227, 5, 49, 0, 0, 227, 229, 1, 0, 0, 0, 228, 221, 1, 0, 0, 0, 228, 222, 1, 0, 0, 0, 228, 223, 1, 0, 0, 0, 228, 224, 1, 0, 0, 0, 229, 19, 1, 0, 0, 0, 230, 231, 3, 48, 24, 0, 231, 241, 5, 39, 0, 0, 232, 242, 5, 60, 0, 0, 233, 238, 3, 10, 5, 0, 234, 235, 5, 33, 0, 0, 235, 237, 3, 10, 5, 0, 236, 234, 1, 0, 0, 0, 237, 240, 1, 0, 0, 0, 238, 236, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 242, 1, 0, 0, 0, 240, 238, 1, 0, 0, 0, 241, 232, 1, 0, 0, 0, 241, 233, 1, 0, 0, 0, 241, 242, 1, 0, 0, 0, 242, 243, 1, 0, 0, 0, 243, 244, 5, 49, 0, 0, 244, 21, 1, 0, 0, 0, 245, 246, 5, 13, 0, 0, 246, 247, 3, 24, 12, 0, 247, 23, 1, 0, 0, 0, 248, 253, 3, 26, 13, 0, 249, 250, 5, 33, 0, 0, 250, 252, 3, 26, 13, 0, 251, 249, 1, 0, 0, 0, 252, 255, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 253, 254, 1, 0, 0, 0, 254, 25, 1, 0, 0, 0, 255, 253, 1, 0, 0, 0, 256, 262, 3, 10, 5, 0, 257, 258, 3, 44, 22, 0, 258, 259, 5, 32, 0, 0, 259, 260, 3, 10, 5, 0, 260, 262, 1, 0, 0, 0, 261, 256, 1, 0, 0, 0, 261, 257, 1, 0, 0, 0, 262, 27, 1, 0, 0, 0, 263, 264, 5, 6, 0, 0, 264, 269, 3, 42, 21, 0, 265, 266, 5, 33, 0, 0, 266, 268, 3, 42, 21, 0, 267, 265, 1, 0, 0, 0, 268, 271, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 273, 1, 0, 0, 0, 271, 269, 1, 0, 0, 0, 272, 274, 3, 30, 15, 0, 273, 272, 1, 0, 0, 0, 273, 274, 1, 0, 0, 0, 274, 29, 1, 0, 0, 0, 275, 278, 3, 32, 16, 0, 276, 278, 3, 34, 17, 0, 277, 275, 1, 0, 0, 0, 277, 276, 1, 0, 0, 0, 278, 31, 1, 0, 0, 0, 279, 280, 5, 70, 0, 0, 280, 285, 3, 42, 21, 0, 281, 282, 5, 33, 0, 0, 282, 284, 3, 42, 21, 0, 283, 281, 1, 0, 0, 0, 284, 287, 1, 0, 0, 0, 285, 283, 1, 0, 0, 0, 285, 286, 1, 0, 0, 0, 286, 33, 1, 0, 0, 0, 287, 285, 1, 0, 0, 0, 288, 289, 5, 63, 0, 0, 289, 290, 3, 32, 16, 0, 290, 291, 5, 64, 0, 0, 291, 35, 1, 0, 0, 0, 292, 293, 5, 4, 0, 0, 293, 294, 3, 24, 12, 0, 294, 37, 1, 0, 0, 0, 295, 297, 5, 16, 0, 0, 296, 298, 3, 24, 12, 0, 297, 296, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 301, 1, 0, 0, 0, 299, 300, 5, 29, 0, 0, 300, 302, 3, 24, 12, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 39, 1, 0, 0, 0, 303, 304, 5, 8, 0, 0, 304, 307, 3, 24, 12, 0, 305, 306, 5, 29, 0, 0, 306, 308, 3, 24, 12, 0, 307, 305, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 41, 1, 0, 0, 0, 309, 310, 7, 2, 0, 0, 310, 43, 1, 0, 0, 0, 311, 316, 3, 48, 24, 0, 312, 313, 5, 35, 0, 0, 313, 315, 3, 48, 24, 0, 314, 312, 1, 0, 0, 0, 315, 318, 1, 0, 0, 0, 316, 314, 1, 0, 0, 0, 316, 317, 1, 0, 0, 0, 317, 45, 1, 0, 0, 0, 318, 316, 1, 0, 0, 0, 319, 324, 3, 50, 25, 0, 320, 321, 5, 35, 0, 0, 321, 323, 3, 50, 25, 0, 322, 320, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 47, 1, 0, 0, 0, 326, 324, 1, 0, 0, 0, 327, 328, 7, 3, 0, 0, 328, 49, 1, 0, 0, 0, 329, 331, 3, 52, 26, 0, 330, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 330, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 51, 1, 0, 0, 0, 334, 335, 7, 4, 0, 0, 335, 53, 1, 0, 0, 0, 336, 379, 5, 44, 0, 0, 337, 338, 3, 86, 43, 0, 338, 339, 5, 65, 0, 0, 339, 379, 1, 0, 0, 0, 340, 379, 3, 84, 42, 0, 341, 379, 3, 86, 43, 0, 342, 379, 3, 80, 40, 0, 343, 379, 5, 47, 0, 0, 344, 379, 3, 88, 44, 0, 345, 346, 5, 63, 0, 0, 346, 351, 3, 82, 41, 0, 347, 348, 5, 33, 0, 0, 348, 350, 3, 82, 41, 0, 349, 347, 1, 0, 0, 0, 350, 353, 1, 0, 0, 0, 351, 349, 1, 0, 0, 0, 351, 352, 1, 0, 0, 0, 352, 354, 1, 0, 0, 0, 353, 351, 1, 0, 0, 0, 354, 355, 5, 64, 0, 0, 355, 379, 1, 0, 0, 0, 356, 357, 5, 63, 0, 0, 357, 362, 3, 80, 40, 0, 358, 359, 5, 33, 0, 0, 359, 361, 3, 80, 40, 0, 360, 358, 1, 0, 0, 0, 361, 364, 1, 0, 0, 0, 362, 360, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 365, 1, 0, 0, 0, 364, 362, 1, 0, 0, 0, 365, 366, 5, 64, 0, 0, 366, 379, 1, 0, 0, 0, 367, 368, 5, 63, 0, 0, 368, 373, 3, 88, 44, 0, 369, 370, 5, 33, 0, 0, 370, 372, 3, 88, 44, 0, 371, 369, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 371, 1, 0, 0, 0, 373, 374, 1, 0, 0, 0, 374, 376, 1, 0, 0, 0, 375, 373, 1, 0, 0, 0, 376, 377, 5, 64, 0, 0, 377, 379, 1, 0, 0, 0, 378, 336, 1, 0, 0, 0, 378, 337, 1, 0, 0, 0, 378, 340, 1, 0, 0, 0, 378, 341, 1, 0, 0, 0, 378, 342, 1, 0, 0, 0, 378, 343, 1, 0, 0, 0, 378, 344, 1, 0, 0, 0, 378, 345, 1, 0, 0, 0, 378, 356, 1, 0, 0, 0, 378, 367, 1, 0, 0, 0, 379, 55, 1, 0, 0, 0, 380, 381, 5, 10, 0, 0, 381, 382, 5, 27, 0, 0, 382, 57, 1, 0, 0, 0, 383, 384, 5, 15, 0, 0, 384, 389, 3, 60, 30, 0, 385, 386, 5, 33, 0, 0, 386, 388, 3, 60, 30, 0, 387, 385, 1, 0, 0, 0, 388, 391, 1, 0, 0, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 59, 1, 0, 0, 0, 391, 389, 1, 0, 0, 0, 392, 394, 3, 10, 5, 0, 393, 395, 7, 5, 0, 0, 394, 393, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 398, 1, 0, 0, 0, 396, 397, 5, 45, 0, 0, 397, 399, 7, 6, 0, 0, 398, 396, 1, 0, 0, 0, 398, 399, 1, 0, 0, 0, 399, 61, 1, 0, 0, 0, 400, 401, 5, 9, 0, 0, 401, 406, 3, 46, 23, 0, 402, 403, 5, 33, 0, 0, 403, 405, 3, 46, 23, 0, 404, 402, 1, 0, 0, 0, 405, 408, 1, 0, 0, 0, 406, 404, 1, 0, 0, 0, 406, 407, 1, 0, 0, 0, 407, 63, 1, 0, 0, 0, 408, 406, 1, 0, 0, 0, 409, 410, 5, 2, 0, 0, 410, 415, 3, 46, 23, 0, 411, 412, 5, 33, 0, 0, 412, 414, 3, 46, 23, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 65, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 419, 5, 12, 0, 0, 419, 424, 3, 68, 34, 0, 420, 421, 5, 33, 0, 0, 421, 423, 3, 68, 34, 0, 422, 420, 1, 0, 0, 0, 423, 426, 1, 0, 0, 0, 424, 422, 1, 0, 0, 0, 424, 425, 1, 0, 0, 0, 425, 67, 1, 0, 0, 0, 426, 424, 1, 0, 0, 0, 427, 428, 3, 46, 23, 0, 428, 429, 5, 79, 0, 0, 429, 430, 3, 46, 23, 0, 430, 69, 1, 0, 0, 0, 431, 432, 5, 1, 0, 0, 432, 433, 3, 18, 9, 0, 433, 435, 3, 88, 44, 0, 434, 436, 3, 76, 38, 0, 435, 434, 1, 0, 0, 0, 435, 436, 1, 0, 0, 0, 436, 71, 1, 0, 0, 0, 437, 438, 5, 7, 0, 0, 438, 439, 3, 18, 9, 0, 439, 440, 3, 88, 44, 0, 440, 73, 1, 0, 0, 0, 441, 442, 5, 11, 0, 0, 442, 443, 3, 44, 22, 0, 443, 75, 1, 0, 0, 0, 444, 449, 3, 78, 39, 0, 445, 446, 5, 33, 0, 0, 446, 448, 3, 78, 39, 0, 447, 445, 1, 0, 0, 0, 448, 451, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 449, 450, 1, 0, 0, 0, 450, 77, 1, 0, 0, 0, 451, 449, 1, 0, 0, 0, 452, 453, 3, 48, 24, 0, 453, 454, 5, 32, 0, 0, 454, 455, 3, 54, 27, 0, 455, 79, 1, 0, 0, 0, 456, 457, 7, 7, 0, 0, 457, 81, 1, 0, 0, 0, 458, 461, 3, 84, 42, 0, 459, 461, 3, 86, 43, 0, 460, 458, 1, 0, 0, 0, 460, 459, 1, 0, 0, 0, 461, 83, 1, 0, 0, 0, 462, 464, 7, 0, 0, 0, 463, 462, 1, 0, 0, 0, 463, 464, 1, 0, 0, 0, 464, 465, 1, 0, 0, 0, 465, 466, 5, 28, 0, 0, 466, 85, 1, 0, 0, 0, 467, 469, 7, 0, 0, 0, 468, 467, 1, 0, 0, 0, 468, 469, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 5, 27, 0, 0, 471, 87, 1, 0, 0, 0, 472, 473, 5, 26, 0, 0, 473, 89, 1, 0, 0, 0, 474, 475, 7, 8, 0, 0, 475, 91, 1, 0, 0, 0, 476, 477, 5, 5, 0, 0, 477, 478, 3, 94, 47, 0, 478, 93, 1, 0, 0, 0, 479, 480, 5, 63, 0, 0, 480, 481, 3, 2, 1, 0, 481, 482, 5, 64, 0, 0, 482, 95, 1, 0, 0, 0, 483, 484, 5, 14, 0, 0, 484, 488, 5, 95, 0, 0, 485, 486, 5, 14, 0, 0, 486, 488, 5, 96, 0, 0, 487, 483, 1, 0, 0, 0, 487, 485, 1, 0, 0, 0, 488, 97, 1, 0, 0, 0, 489, 490, 5, 3, 0, 0, 490, 493, 5, 85, 0, 0, 491, 492, 5, 83, 0, 0, 492, 494, 3, 46, 23, 0, 493, 491, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 504, 1, 0, 0, 0, 495, 496, 5, 84, 0, 0, 496, 501, 3, 100, 50, 0, 497, 498, 5, 33, 0, 0, 498, 500, 3, 100, 50, 0, 499, 497, 1, 0, 0, 0, 500, 503, 1, 0, 0, 0, 501, 499, 1, 0, 0, 0, 501, 502, 1, 0, 0, 0, 502, 505, 1, 0, 0, 0, 503, 501, 1, 0, 0, 0, 504, 495, 1, 0, 0, 0, 504, 505, 1, 0, 0, 0, 505, 99, 1, 0, 0, 0, 506, 507, 3, 46, 23, 0, 507, 508, 5, 32, 0, 0, 508, 510, 1, 0, 0, 0, 509, 506, 1, 0, 0, 0, 509, 510, 1, 0, 0, 0, 510, 511, 1, 0, 0, 0, 511, 512, 3, 46, 23, 0, 512, 101, 1, 0, 0, 0, 51, 113, 120, 135, 147, 156, 164, 168, 176, 178, 183, 190, 195, 202, 208, 216, 218, 228, 238, 241, 253, 261, 269, 273, 277, 285, 297, 301, 307, 316, 324, 332, 351, 362, 373, 378, 389, 394, 398, 406, 415, 424, 435, 449, 460, 463, 468, 487, 493, 501, 504, 509] \ No newline at end of file +[4, 1, 104, 507, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 110, 8, 1, 10, 1, 12, 1, 113, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 119, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 134, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 146, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 153, 8, 5, 10, 5, 12, 5, 156, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 163, 8, 5, 1, 5, 1, 5, 3, 5, 167, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 175, 8, 5, 10, 5, 12, 5, 178, 9, 5, 1, 6, 1, 6, 3, 6, 182, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 189, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 194, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 201, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 207, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 215, 8, 8, 10, 8, 12, 8, 218, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 227, 8, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 235, 8, 10, 10, 10, 12, 10, 238, 9, 10, 3, 10, 240, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 5, 12, 250, 8, 12, 10, 12, 12, 12, 253, 9, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 260, 8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 5, 14, 266, 8, 14, 10, 14, 12, 14, 269, 9, 14, 1, 14, 3, 14, 272, 8, 14, 1, 15, 1, 15, 3, 15, 276, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 282, 8, 16, 10, 16, 12, 16, 285, 9, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 3, 19, 296, 8, 19, 1, 19, 1, 19, 3, 19, 300, 8, 19, 1, 20, 1, 20, 1, 20, 1, 20, 3, 20, 306, 8, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 5, 22, 313, 8, 22, 10, 22, 12, 22, 316, 9, 22, 1, 23, 1, 23, 1, 23, 5, 23, 321, 8, 23, 10, 23, 12, 23, 324, 9, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 343, 8, 26, 10, 26, 12, 26, 346, 9, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 354, 8, 26, 10, 26, 12, 26, 357, 9, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 5, 26, 365, 8, 26, 10, 26, 12, 26, 368, 9, 26, 1, 26, 1, 26, 3, 26, 372, 8, 26, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 381, 8, 28, 10, 28, 12, 28, 384, 9, 28, 1, 29, 1, 29, 3, 29, 388, 8, 29, 1, 29, 1, 29, 3, 29, 392, 8, 29, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 398, 8, 30, 10, 30, 12, 30, 401, 9, 30, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 407, 8, 31, 10, 31, 12, 31, 410, 9, 31, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 416, 8, 32, 10, 32, 12, 32, 419, 9, 32, 1, 33, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 429, 8, 34, 1, 35, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 5, 37, 441, 8, 37, 10, 37, 12, 37, 444, 9, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 40, 1, 40, 3, 40, 454, 8, 40, 1, 41, 3, 41, 457, 8, 41, 1, 41, 1, 41, 1, 42, 3, 42, 462, 8, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 3, 47, 481, 8, 47, 1, 48, 1, 48, 1, 48, 1, 48, 3, 48, 487, 8, 48, 1, 48, 1, 48, 1, 48, 1, 48, 5, 48, 493, 8, 48, 10, 48, 12, 48, 496, 9, 48, 3, 48, 498, 8, 48, 1, 49, 1, 49, 1, 49, 3, 49, 503, 8, 49, 1, 49, 1, 49, 1, 49, 0, 3, 2, 10, 16, 50, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 0, 8, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 66, 66, 71, 71, 1, 0, 65, 66, 2, 0, 31, 31, 34, 34, 1, 0, 37, 38, 2, 0, 36, 36, 50, 50, 1, 0, 51, 57, 533, 0, 100, 1, 0, 0, 0, 2, 103, 1, 0, 0, 0, 4, 118, 1, 0, 0, 0, 6, 133, 1, 0, 0, 0, 8, 135, 1, 0, 0, 0, 10, 166, 1, 0, 0, 0, 12, 193, 1, 0, 0, 0, 14, 200, 1, 0, 0, 0, 16, 206, 1, 0, 0, 0, 18, 226, 1, 0, 0, 0, 20, 228, 1, 0, 0, 0, 22, 243, 1, 0, 0, 0, 24, 246, 1, 0, 0, 0, 26, 259, 1, 0, 0, 0, 28, 261, 1, 0, 0, 0, 30, 275, 1, 0, 0, 0, 32, 277, 1, 0, 0, 0, 34, 286, 1, 0, 0, 0, 36, 290, 1, 0, 0, 0, 38, 293, 1, 0, 0, 0, 40, 301, 1, 0, 0, 0, 42, 307, 1, 0, 0, 0, 44, 309, 1, 0, 0, 0, 46, 317, 1, 0, 0, 0, 48, 325, 1, 0, 0, 0, 50, 327, 1, 0, 0, 0, 52, 371, 1, 0, 0, 0, 54, 373, 1, 0, 0, 0, 56, 376, 1, 0, 0, 0, 58, 385, 1, 0, 0, 0, 60, 393, 1, 0, 0, 0, 62, 402, 1, 0, 0, 0, 64, 411, 1, 0, 0, 0, 66, 420, 1, 0, 0, 0, 68, 424, 1, 0, 0, 0, 70, 430, 1, 0, 0, 0, 72, 434, 1, 0, 0, 0, 74, 437, 1, 0, 0, 0, 76, 445, 1, 0, 0, 0, 78, 449, 1, 0, 0, 0, 80, 453, 1, 0, 0, 0, 82, 456, 1, 0, 0, 0, 84, 461, 1, 0, 0, 0, 86, 465, 1, 0, 0, 0, 88, 467, 1, 0, 0, 0, 90, 469, 1, 0, 0, 0, 92, 472, 1, 0, 0, 0, 94, 480, 1, 0, 0, 0, 96, 482, 1, 0, 0, 0, 98, 502, 1, 0, 0, 0, 100, 101, 3, 2, 1, 0, 101, 102, 5, 0, 0, 1, 102, 1, 1, 0, 0, 0, 103, 104, 6, 1, -1, 0, 104, 105, 3, 4, 2, 0, 105, 111, 1, 0, 0, 0, 106, 107, 10, 1, 0, 0, 107, 108, 5, 25, 0, 0, 108, 110, 3, 6, 3, 0, 109, 106, 1, 0, 0, 0, 110, 113, 1, 0, 0, 0, 111, 109, 1, 0, 0, 0, 111, 112, 1, 0, 0, 0, 112, 3, 1, 0, 0, 0, 113, 111, 1, 0, 0, 0, 114, 119, 3, 90, 45, 0, 115, 119, 3, 28, 14, 0, 116, 119, 3, 22, 11, 0, 117, 119, 3, 94, 47, 0, 118, 114, 1, 0, 0, 0, 118, 115, 1, 0, 0, 0, 118, 116, 1, 0, 0, 0, 118, 117, 1, 0, 0, 0, 119, 5, 1, 0, 0, 0, 120, 134, 3, 36, 18, 0, 121, 134, 3, 40, 20, 0, 122, 134, 3, 54, 27, 0, 123, 134, 3, 60, 30, 0, 124, 134, 3, 56, 28, 0, 125, 134, 3, 38, 19, 0, 126, 134, 3, 8, 4, 0, 127, 134, 3, 62, 31, 0, 128, 134, 3, 64, 32, 0, 129, 134, 3, 68, 34, 0, 130, 134, 3, 70, 35, 0, 131, 134, 3, 96, 48, 0, 132, 134, 3, 72, 36, 0, 133, 120, 1, 0, 0, 0, 133, 121, 1, 0, 0, 0, 133, 122, 1, 0, 0, 0, 133, 123, 1, 0, 0, 0, 133, 124, 1, 0, 0, 0, 133, 125, 1, 0, 0, 0, 133, 126, 1, 0, 0, 0, 133, 127, 1, 0, 0, 0, 133, 128, 1, 0, 0, 0, 133, 129, 1, 0, 0, 0, 133, 130, 1, 0, 0, 0, 133, 131, 1, 0, 0, 0, 133, 132, 1, 0, 0, 0, 134, 7, 1, 0, 0, 0, 135, 136, 5, 17, 0, 0, 136, 137, 3, 10, 5, 0, 137, 9, 1, 0, 0, 0, 138, 139, 6, 5, -1, 0, 139, 140, 5, 43, 0, 0, 140, 167, 3, 10, 5, 7, 141, 167, 3, 14, 7, 0, 142, 167, 3, 12, 6, 0, 143, 145, 3, 14, 7, 0, 144, 146, 5, 43, 0, 0, 145, 144, 1, 0, 0, 0, 145, 146, 1, 0, 0, 0, 146, 147, 1, 0, 0, 0, 147, 148, 5, 40, 0, 0, 148, 149, 5, 39, 0, 0, 149, 154, 3, 14, 7, 0, 150, 151, 5, 33, 0, 0, 151, 153, 3, 14, 7, 0, 152, 150, 1, 0, 0, 0, 153, 156, 1, 0, 0, 0, 154, 152, 1, 0, 0, 0, 154, 155, 1, 0, 0, 0, 155, 157, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 157, 158, 5, 49, 0, 0, 158, 167, 1, 0, 0, 0, 159, 160, 3, 14, 7, 0, 160, 162, 5, 41, 0, 0, 161, 163, 5, 43, 0, 0, 162, 161, 1, 0, 0, 0, 162, 163, 1, 0, 0, 0, 163, 164, 1, 0, 0, 0, 164, 165, 5, 44, 0, 0, 165, 167, 1, 0, 0, 0, 166, 138, 1, 0, 0, 0, 166, 141, 1, 0, 0, 0, 166, 142, 1, 0, 0, 0, 166, 143, 1, 0, 0, 0, 166, 159, 1, 0, 0, 0, 167, 176, 1, 0, 0, 0, 168, 169, 10, 4, 0, 0, 169, 170, 5, 30, 0, 0, 170, 175, 3, 10, 5, 5, 171, 172, 10, 3, 0, 0, 172, 173, 5, 46, 0, 0, 173, 175, 3, 10, 5, 4, 174, 168, 1, 0, 0, 0, 174, 171, 1, 0, 0, 0, 175, 178, 1, 0, 0, 0, 176, 174, 1, 0, 0, 0, 176, 177, 1, 0, 0, 0, 177, 11, 1, 0, 0, 0, 178, 176, 1, 0, 0, 0, 179, 181, 3, 14, 7, 0, 180, 182, 5, 43, 0, 0, 181, 180, 1, 0, 0, 0, 181, 182, 1, 0, 0, 0, 182, 183, 1, 0, 0, 0, 183, 184, 5, 42, 0, 0, 184, 185, 3, 86, 43, 0, 185, 194, 1, 0, 0, 0, 186, 188, 3, 14, 7, 0, 187, 189, 5, 43, 0, 0, 188, 187, 1, 0, 0, 0, 188, 189, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 191, 5, 48, 0, 0, 191, 192, 3, 86, 43, 0, 192, 194, 1, 0, 0, 0, 193, 179, 1, 0, 0, 0, 193, 186, 1, 0, 0, 0, 194, 13, 1, 0, 0, 0, 195, 201, 3, 16, 8, 0, 196, 197, 3, 16, 8, 0, 197, 198, 3, 88, 44, 0, 198, 199, 3, 16, 8, 0, 199, 201, 1, 0, 0, 0, 200, 195, 1, 0, 0, 0, 200, 196, 1, 0, 0, 0, 201, 15, 1, 0, 0, 0, 202, 203, 6, 8, -1, 0, 203, 207, 3, 18, 9, 0, 204, 205, 7, 0, 0, 0, 205, 207, 3, 16, 8, 3, 206, 202, 1, 0, 0, 0, 206, 204, 1, 0, 0, 0, 207, 216, 1, 0, 0, 0, 208, 209, 10, 2, 0, 0, 209, 210, 7, 1, 0, 0, 210, 215, 3, 16, 8, 3, 211, 212, 10, 1, 0, 0, 212, 213, 7, 0, 0, 0, 213, 215, 3, 16, 8, 2, 214, 208, 1, 0, 0, 0, 214, 211, 1, 0, 0, 0, 215, 218, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 17, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 219, 227, 3, 52, 26, 0, 220, 227, 3, 44, 22, 0, 221, 227, 3, 20, 10, 0, 222, 223, 5, 39, 0, 0, 223, 224, 3, 10, 5, 0, 224, 225, 5, 49, 0, 0, 225, 227, 1, 0, 0, 0, 226, 219, 1, 0, 0, 0, 226, 220, 1, 0, 0, 0, 226, 221, 1, 0, 0, 0, 226, 222, 1, 0, 0, 0, 227, 19, 1, 0, 0, 0, 228, 229, 3, 48, 24, 0, 229, 239, 5, 39, 0, 0, 230, 240, 5, 60, 0, 0, 231, 236, 3, 10, 5, 0, 232, 233, 5, 33, 0, 0, 233, 235, 3, 10, 5, 0, 234, 232, 1, 0, 0, 0, 235, 238, 1, 0, 0, 0, 236, 234, 1, 0, 0, 0, 236, 237, 1, 0, 0, 0, 237, 240, 1, 0, 0, 0, 238, 236, 1, 0, 0, 0, 239, 230, 1, 0, 0, 0, 239, 231, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 241, 1, 0, 0, 0, 241, 242, 5, 49, 0, 0, 242, 21, 1, 0, 0, 0, 243, 244, 5, 13, 0, 0, 244, 245, 3, 24, 12, 0, 245, 23, 1, 0, 0, 0, 246, 251, 3, 26, 13, 0, 247, 248, 5, 33, 0, 0, 248, 250, 3, 26, 13, 0, 249, 247, 1, 0, 0, 0, 250, 253, 1, 0, 0, 0, 251, 249, 1, 0, 0, 0, 251, 252, 1, 0, 0, 0, 252, 25, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 254, 260, 3, 10, 5, 0, 255, 256, 3, 44, 22, 0, 256, 257, 5, 32, 0, 0, 257, 258, 3, 10, 5, 0, 258, 260, 1, 0, 0, 0, 259, 254, 1, 0, 0, 0, 259, 255, 1, 0, 0, 0, 260, 27, 1, 0, 0, 0, 261, 262, 5, 6, 0, 0, 262, 267, 3, 42, 21, 0, 263, 264, 5, 33, 0, 0, 264, 266, 3, 42, 21, 0, 265, 263, 1, 0, 0, 0, 266, 269, 1, 0, 0, 0, 267, 265, 1, 0, 0, 0, 267, 268, 1, 0, 0, 0, 268, 271, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 270, 272, 3, 30, 15, 0, 271, 270, 1, 0, 0, 0, 271, 272, 1, 0, 0, 0, 272, 29, 1, 0, 0, 0, 273, 276, 3, 32, 16, 0, 274, 276, 3, 34, 17, 0, 275, 273, 1, 0, 0, 0, 275, 274, 1, 0, 0, 0, 276, 31, 1, 0, 0, 0, 277, 278, 5, 70, 0, 0, 278, 283, 3, 42, 21, 0, 279, 280, 5, 33, 0, 0, 280, 282, 3, 42, 21, 0, 281, 279, 1, 0, 0, 0, 282, 285, 1, 0, 0, 0, 283, 281, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 33, 1, 0, 0, 0, 285, 283, 1, 0, 0, 0, 286, 287, 5, 63, 0, 0, 287, 288, 3, 32, 16, 0, 288, 289, 5, 64, 0, 0, 289, 35, 1, 0, 0, 0, 290, 291, 5, 4, 0, 0, 291, 292, 3, 24, 12, 0, 292, 37, 1, 0, 0, 0, 293, 295, 5, 16, 0, 0, 294, 296, 3, 24, 12, 0, 295, 294, 1, 0, 0, 0, 295, 296, 1, 0, 0, 0, 296, 299, 1, 0, 0, 0, 297, 298, 5, 29, 0, 0, 298, 300, 3, 24, 12, 0, 299, 297, 1, 0, 0, 0, 299, 300, 1, 0, 0, 0, 300, 39, 1, 0, 0, 0, 301, 302, 5, 8, 0, 0, 302, 305, 3, 24, 12, 0, 303, 304, 5, 29, 0, 0, 304, 306, 3, 24, 12, 0, 305, 303, 1, 0, 0, 0, 305, 306, 1, 0, 0, 0, 306, 41, 1, 0, 0, 0, 307, 308, 7, 2, 0, 0, 308, 43, 1, 0, 0, 0, 309, 314, 3, 48, 24, 0, 310, 311, 5, 35, 0, 0, 311, 313, 3, 48, 24, 0, 312, 310, 1, 0, 0, 0, 313, 316, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 45, 1, 0, 0, 0, 316, 314, 1, 0, 0, 0, 317, 322, 3, 50, 25, 0, 318, 319, 5, 35, 0, 0, 319, 321, 3, 50, 25, 0, 320, 318, 1, 0, 0, 0, 321, 324, 1, 0, 0, 0, 322, 320, 1, 0, 0, 0, 322, 323, 1, 0, 0, 0, 323, 47, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 325, 326, 7, 3, 0, 0, 326, 49, 1, 0, 0, 0, 327, 328, 5, 75, 0, 0, 328, 51, 1, 0, 0, 0, 329, 372, 5, 44, 0, 0, 330, 331, 3, 84, 42, 0, 331, 332, 5, 65, 0, 0, 332, 372, 1, 0, 0, 0, 333, 372, 3, 82, 41, 0, 334, 372, 3, 84, 42, 0, 335, 372, 3, 78, 39, 0, 336, 372, 5, 47, 0, 0, 337, 372, 3, 86, 43, 0, 338, 339, 5, 63, 0, 0, 339, 344, 3, 80, 40, 0, 340, 341, 5, 33, 0, 0, 341, 343, 3, 80, 40, 0, 342, 340, 1, 0, 0, 0, 343, 346, 1, 0, 0, 0, 344, 342, 1, 0, 0, 0, 344, 345, 1, 0, 0, 0, 345, 347, 1, 0, 0, 0, 346, 344, 1, 0, 0, 0, 347, 348, 5, 64, 0, 0, 348, 372, 1, 0, 0, 0, 349, 350, 5, 63, 0, 0, 350, 355, 3, 78, 39, 0, 351, 352, 5, 33, 0, 0, 352, 354, 3, 78, 39, 0, 353, 351, 1, 0, 0, 0, 354, 357, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 358, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 358, 359, 5, 64, 0, 0, 359, 372, 1, 0, 0, 0, 360, 361, 5, 63, 0, 0, 361, 366, 3, 86, 43, 0, 362, 363, 5, 33, 0, 0, 363, 365, 3, 86, 43, 0, 364, 362, 1, 0, 0, 0, 365, 368, 1, 0, 0, 0, 366, 364, 1, 0, 0, 0, 366, 367, 1, 0, 0, 0, 367, 369, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 369, 370, 5, 64, 0, 0, 370, 372, 1, 0, 0, 0, 371, 329, 1, 0, 0, 0, 371, 330, 1, 0, 0, 0, 371, 333, 1, 0, 0, 0, 371, 334, 1, 0, 0, 0, 371, 335, 1, 0, 0, 0, 371, 336, 1, 0, 0, 0, 371, 337, 1, 0, 0, 0, 371, 338, 1, 0, 0, 0, 371, 349, 1, 0, 0, 0, 371, 360, 1, 0, 0, 0, 372, 53, 1, 0, 0, 0, 373, 374, 5, 10, 0, 0, 374, 375, 5, 27, 0, 0, 375, 55, 1, 0, 0, 0, 376, 377, 5, 15, 0, 0, 377, 382, 3, 58, 29, 0, 378, 379, 5, 33, 0, 0, 379, 381, 3, 58, 29, 0, 380, 378, 1, 0, 0, 0, 381, 384, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 57, 1, 0, 0, 0, 384, 382, 1, 0, 0, 0, 385, 387, 3, 10, 5, 0, 386, 388, 7, 4, 0, 0, 387, 386, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 391, 1, 0, 0, 0, 389, 390, 5, 45, 0, 0, 390, 392, 7, 5, 0, 0, 391, 389, 1, 0, 0, 0, 391, 392, 1, 0, 0, 0, 392, 59, 1, 0, 0, 0, 393, 394, 5, 9, 0, 0, 394, 399, 3, 46, 23, 0, 395, 396, 5, 33, 0, 0, 396, 398, 3, 46, 23, 0, 397, 395, 1, 0, 0, 0, 398, 401, 1, 0, 0, 0, 399, 397, 1, 0, 0, 0, 399, 400, 1, 0, 0, 0, 400, 61, 1, 0, 0, 0, 401, 399, 1, 0, 0, 0, 402, 403, 5, 2, 0, 0, 403, 408, 3, 46, 23, 0, 404, 405, 5, 33, 0, 0, 405, 407, 3, 46, 23, 0, 406, 404, 1, 0, 0, 0, 407, 410, 1, 0, 0, 0, 408, 406, 1, 0, 0, 0, 408, 409, 1, 0, 0, 0, 409, 63, 1, 0, 0, 0, 410, 408, 1, 0, 0, 0, 411, 412, 5, 12, 0, 0, 412, 417, 3, 66, 33, 0, 413, 414, 5, 33, 0, 0, 414, 416, 3, 66, 33, 0, 415, 413, 1, 0, 0, 0, 416, 419, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 417, 418, 1, 0, 0, 0, 418, 65, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 420, 421, 3, 46, 23, 0, 421, 422, 5, 79, 0, 0, 422, 423, 3, 46, 23, 0, 423, 67, 1, 0, 0, 0, 424, 425, 5, 1, 0, 0, 425, 426, 3, 18, 9, 0, 426, 428, 3, 86, 43, 0, 427, 429, 3, 74, 37, 0, 428, 427, 1, 0, 0, 0, 428, 429, 1, 0, 0, 0, 429, 69, 1, 0, 0, 0, 430, 431, 5, 7, 0, 0, 431, 432, 3, 18, 9, 0, 432, 433, 3, 86, 43, 0, 433, 71, 1, 0, 0, 0, 434, 435, 5, 11, 0, 0, 435, 436, 3, 44, 22, 0, 436, 73, 1, 0, 0, 0, 437, 442, 3, 76, 38, 0, 438, 439, 5, 33, 0, 0, 439, 441, 3, 76, 38, 0, 440, 438, 1, 0, 0, 0, 441, 444, 1, 0, 0, 0, 442, 440, 1, 0, 0, 0, 442, 443, 1, 0, 0, 0, 443, 75, 1, 0, 0, 0, 444, 442, 1, 0, 0, 0, 445, 446, 3, 48, 24, 0, 446, 447, 5, 32, 0, 0, 447, 448, 3, 52, 26, 0, 448, 77, 1, 0, 0, 0, 449, 450, 7, 6, 0, 0, 450, 79, 1, 0, 0, 0, 451, 454, 3, 82, 41, 0, 452, 454, 3, 84, 42, 0, 453, 451, 1, 0, 0, 0, 453, 452, 1, 0, 0, 0, 454, 81, 1, 0, 0, 0, 455, 457, 7, 0, 0, 0, 456, 455, 1, 0, 0, 0, 456, 457, 1, 0, 0, 0, 457, 458, 1, 0, 0, 0, 458, 459, 5, 28, 0, 0, 459, 83, 1, 0, 0, 0, 460, 462, 7, 0, 0, 0, 461, 460, 1, 0, 0, 0, 461, 462, 1, 0, 0, 0, 462, 463, 1, 0, 0, 0, 463, 464, 5, 27, 0, 0, 464, 85, 1, 0, 0, 0, 465, 466, 5, 26, 0, 0, 466, 87, 1, 0, 0, 0, 467, 468, 7, 7, 0, 0, 468, 89, 1, 0, 0, 0, 469, 470, 5, 5, 0, 0, 470, 471, 3, 92, 46, 0, 471, 91, 1, 0, 0, 0, 472, 473, 5, 63, 0, 0, 473, 474, 3, 2, 1, 0, 474, 475, 5, 64, 0, 0, 475, 93, 1, 0, 0, 0, 476, 477, 5, 14, 0, 0, 477, 481, 5, 95, 0, 0, 478, 479, 5, 14, 0, 0, 479, 481, 5, 96, 0, 0, 480, 476, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 481, 95, 1, 0, 0, 0, 482, 483, 5, 3, 0, 0, 483, 486, 5, 85, 0, 0, 484, 485, 5, 83, 0, 0, 485, 487, 3, 46, 23, 0, 486, 484, 1, 0, 0, 0, 486, 487, 1, 0, 0, 0, 487, 497, 1, 0, 0, 0, 488, 489, 5, 84, 0, 0, 489, 494, 3, 98, 49, 0, 490, 491, 5, 33, 0, 0, 491, 493, 3, 98, 49, 0, 492, 490, 1, 0, 0, 0, 493, 496, 1, 0, 0, 0, 494, 492, 1, 0, 0, 0, 494, 495, 1, 0, 0, 0, 495, 498, 1, 0, 0, 0, 496, 494, 1, 0, 0, 0, 497, 488, 1, 0, 0, 0, 497, 498, 1, 0, 0, 0, 498, 97, 1, 0, 0, 0, 499, 500, 3, 46, 23, 0, 500, 501, 5, 32, 0, 0, 501, 503, 1, 0, 0, 0, 502, 499, 1, 0, 0, 0, 502, 503, 1, 0, 0, 0, 503, 504, 1, 0, 0, 0, 504, 505, 3, 46, 23, 0, 505, 99, 1, 0, 0, 0, 50, 111, 118, 133, 145, 154, 162, 166, 174, 176, 181, 188, 193, 200, 206, 214, 216, 226, 236, 239, 251, 259, 267, 271, 275, 283, 295, 299, 305, 314, 322, 344, 355, 366, 371, 382, 387, 391, 399, 408, 417, 428, 442, 453, 456, 461, 480, 486, 494, 497, 502] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index 23154e3b326be..ab83872ce0ac3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -28,7 +28,7 @@ public class EsqlBaseParser extends Parser { GTE=57, PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, OPENING_BRACKET=63, CLOSING_BRACKET=64, UNQUOTED_IDENTIFIER=65, QUOTED_IDENTIFIER=66, EXPR_LINE_COMMENT=67, EXPR_MULTILINE_COMMENT=68, EXPR_WS=69, METADATA=70, FROM_UNQUOTED_IDENTIFIER=71, - FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, UNQUOTED_ID_PATTERN=75, + FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, ID_PATTERN=75, PROJECT_LINE_COMMENT=76, PROJECT_MULTILINE_COMMENT=77, PROJECT_WS=78, AS=79, RENAME_LINE_COMMENT=80, RENAME_MULTILINE_COMMENT=81, RENAME_WS=82, ON=83, WITH=84, ENRICH_POLICY_NAME=85, ENRICH_LINE_COMMENT=86, ENRICH_MULTILINE_COMMENT=87, @@ -46,14 +46,14 @@ public class EsqlBaseParser extends Parser { RULE_deprecated_metadata = 17, RULE_evalCommand = 18, RULE_statsCommand = 19, RULE_inlinestatsCommand = 20, RULE_fromIdentifier = 21, RULE_qualifiedName = 22, RULE_qualifiedNamePattern = 23, RULE_identifier = 24, RULE_identifierPattern = 25, - RULE_idPattern = 26, RULE_constant = 27, RULE_limitCommand = 28, RULE_sortCommand = 29, - RULE_orderExpression = 30, RULE_keepCommand = 31, RULE_dropCommand = 32, - RULE_renameCommand = 33, RULE_renameClause = 34, RULE_dissectCommand = 35, - RULE_grokCommand = 36, RULE_mvExpandCommand = 37, RULE_commandOptions = 38, - RULE_commandOption = 39, RULE_booleanValue = 40, RULE_numericValue = 41, - RULE_decimalValue = 42, RULE_integerValue = 43, RULE_string = 44, RULE_comparisonOperator = 45, - RULE_explainCommand = 46, RULE_subqueryExpression = 47, RULE_showCommand = 48, - RULE_enrichCommand = 49, RULE_enrichWithClause = 50; + RULE_constant = 26, RULE_limitCommand = 27, RULE_sortCommand = 28, RULE_orderExpression = 29, + RULE_keepCommand = 30, RULE_dropCommand = 31, RULE_renameCommand = 32, + RULE_renameClause = 33, RULE_dissectCommand = 34, RULE_grokCommand = 35, + RULE_mvExpandCommand = 36, RULE_commandOptions = 37, RULE_commandOption = 38, + RULE_booleanValue = 39, RULE_numericValue = 40, RULE_decimalValue = 41, + RULE_integerValue = 42, RULE_string = 43, RULE_comparisonOperator = 44, + RULE_explainCommand = 45, RULE_subqueryExpression = 46, RULE_showCommand = 47, + RULE_enrichCommand = 48, RULE_enrichWithClause = 49; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", @@ -61,13 +61,12 @@ private static String[] makeRuleNames() { "primaryExpression", "functionExpression", "rowCommand", "fields", "field", "fromCommand", "metadata", "metadataOption", "deprecated_metadata", "evalCommand", "statsCommand", "inlinestatsCommand", "fromIdentifier", "qualifiedName", - "qualifiedNamePattern", "identifier", "identifierPattern", "idPattern", - "constant", "limitCommand", "sortCommand", "orderExpression", "keepCommand", - "dropCommand", "renameCommand", "renameClause", "dissectCommand", "grokCommand", - "mvExpandCommand", "commandOptions", "commandOption", "booleanValue", - "numericValue", "decimalValue", "integerValue", "string", "comparisonOperator", - "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", - "enrichWithClause" + "qualifiedNamePattern", "identifier", "identifierPattern", "constant", + "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", + "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", + "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", + "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "showCommand", "enrichCommand", "enrichWithClause" }; } public static final String[] ruleNames = makeRuleNames(); @@ -102,7 +101,7 @@ private static String[] makeSymbolicNames() { "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "METADATA", "FROM_UNQUOTED_IDENTIFIER", "FROM_LINE_COMMENT", - "FROM_MULTILINE_COMMENT", "FROM_WS", "UNQUOTED_ID_PATTERN", "PROJECT_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", @@ -196,9 +195,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(102); + setState(100); query(0); - setState(103); + setState(101); match(EOF); } } @@ -294,11 +293,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(106); + setState(104); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(113); + setState(111); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -309,16 +308,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(108); + setState(106); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(109); + setState(107); match(PIPE); - setState(110); + setState(108); processingCommand(); } } } - setState(115); + setState(113); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -373,34 +372,34 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(120); + setState(118); _errHandler.sync(this); switch (_input.LA(1)) { case EXPLAIN: enterOuterAlt(_localctx, 1); { - setState(116); + setState(114); explainCommand(); } break; case FROM: enterOuterAlt(_localctx, 2); { - setState(117); + setState(115); fromCommand(); } break; case ROW: enterOuterAlt(_localctx, 3); { - setState(118); + setState(116); rowCommand(); } break; case SHOW: enterOuterAlt(_localctx, 4); { - setState(119); + setState(117); showCommand(); } break; @@ -484,97 +483,97 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(135); + setState(133); _errHandler.sync(this); switch (_input.LA(1)) { case EVAL: enterOuterAlt(_localctx, 1); { - setState(122); + setState(120); evalCommand(); } break; case INLINESTATS: enterOuterAlt(_localctx, 2); { - setState(123); + setState(121); inlinestatsCommand(); } break; case LIMIT: enterOuterAlt(_localctx, 3); { - setState(124); + setState(122); limitCommand(); } break; case KEEP: enterOuterAlt(_localctx, 4); { - setState(125); + setState(123); keepCommand(); } break; case SORT: enterOuterAlt(_localctx, 5); { - setState(126); + setState(124); sortCommand(); } break; case STATS: enterOuterAlt(_localctx, 6); { - setState(127); + setState(125); statsCommand(); } break; case WHERE: enterOuterAlt(_localctx, 7); { - setState(128); + setState(126); whereCommand(); } break; case DROP: enterOuterAlt(_localctx, 8); { - setState(129); + setState(127); dropCommand(); } break; case RENAME: enterOuterAlt(_localctx, 9); { - setState(130); + setState(128); renameCommand(); } break; case DISSECT: enterOuterAlt(_localctx, 10); { - setState(131); + setState(129); dissectCommand(); } break; case GROK: enterOuterAlt(_localctx, 11); { - setState(132); + setState(130); grokCommand(); } break; case ENRICH: enterOuterAlt(_localctx, 12); { - setState(133); + setState(131); enrichCommand(); } break; case MV_EXPAND: enterOuterAlt(_localctx, 13); { - setState(134); + setState(132); mvExpandCommand(); } break; @@ -625,9 +624,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(137); + setState(135); match(WHERE); - setState(138); + setState(136); booleanExpression(0); } } @@ -822,7 +821,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(168); + setState(166); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -831,9 +830,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(141); + setState(139); match(NOT); - setState(142); + setState(140); booleanExpression(7); } break; @@ -842,7 +841,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(143); + setState(141); valueExpression(); } break; @@ -851,7 +850,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(144); + setState(142); regexBooleanExpression(); } break; @@ -860,41 +859,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(145); + setState(143); valueExpression(); - setState(147); + setState(145); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(146); + setState(144); match(NOT); } } - setState(149); + setState(147); match(IN); - setState(150); + setState(148); match(LP); - setState(151); + setState(149); valueExpression(); - setState(156); + setState(154); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(152); + setState(150); match(COMMA); - setState(153); + setState(151); valueExpression(); } } - setState(158); + setState(156); _errHandler.sync(this); _la = _input.LA(1); } - setState(159); + setState(157); match(RP); } break; @@ -903,27 +902,27 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(161); + setState(159); valueExpression(); - setState(162); + setState(160); match(IS); - setState(164); + setState(162); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(163); + setState(161); match(NOT); } } - setState(166); + setState(164); match(NULL); } break; } _ctx.stop = _input.LT(-1); - setState(178); + setState(176); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -931,7 +930,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(176); + setState(174); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -939,11 +938,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(170); + setState(168); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(171); + setState(169); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(172); + setState(170); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; @@ -952,18 +951,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(173); + setState(171); if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(174); + setState(172); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(175); + setState(173); ((LogicalBinaryContext)_localctx).right = booleanExpression(4); } break; } } } - setState(180); + setState(178); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1018,48 +1017,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(195); + setState(193); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(181); + setState(179); valueExpression(); - setState(183); + setState(181); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(182); + setState(180); match(NOT); } } - setState(185); + setState(183); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(186); + setState(184); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(188); + setState(186); valueExpression(); - setState(190); + setState(188); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(189); + setState(187); match(NOT); } } - setState(192); + setState(190); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(193); + setState(191); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1145,14 +1144,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 14, RULE_valueExpression); try { - setState(202); + setState(200); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(197); + setState(195); operatorExpression(0); } break; @@ -1160,11 +1159,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(198); + setState(196); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(199); + setState(197); comparisonOperator(); - setState(200); + setState(198); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1289,7 +1288,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(208); + setState(206); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: @@ -1298,7 +1297,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(205); + setState(203); primaryExpression(); } break; @@ -1307,7 +1306,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(206); + setState(204); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1318,13 +1317,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(207); + setState(205); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(218); + setState(216); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1332,7 +1331,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(216); + setState(214); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1340,9 +1339,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(210); + setState(208); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(211); + setState(209); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 8070450532247928832L) != 0)) ) { @@ -1353,7 +1352,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(212); + setState(210); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1362,9 +1361,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(213); + setState(211); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(214); + setState(212); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1375,14 +1374,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(215); + setState(213); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(220); + setState(218); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); } @@ -1504,14 +1503,14 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce PrimaryExpressionContext _localctx = new PrimaryExpressionContext(_ctx, getState()); enterRule(_localctx, 18, RULE_primaryExpression); try { - setState(228); + setState(226); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { case 1: _localctx = new ConstantDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(221); + setState(219); constant(); } break; @@ -1519,7 +1518,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new DereferenceContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(222); + setState(220); qualifiedName(); } break; @@ -1527,7 +1526,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new FunctionContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(223); + setState(221); functionExpression(); } break; @@ -1535,11 +1534,11 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new ParenthesizedExpressionContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(224); + setState(222); match(LP); - setState(225); + setState(223); booleanExpression(0); - setState(226); + setState(224); match(RP); } break; @@ -1601,16 +1600,16 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(230); + setState(228); identifier(); - setState(231); + setState(229); match(LP); - setState(241); + setState(239); _errHandler.sync(this); switch (_input.LA(1)) { case ASTERISK: { - setState(232); + setState(230); match(ASTERISK); } break; @@ -1630,21 +1629,21 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx case QUOTED_IDENTIFIER: { { - setState(233); + setState(231); booleanExpression(0); - setState(238); + setState(236); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(234); + setState(232); match(COMMA); - setState(235); + setState(233); booleanExpression(0); } } - setState(240); + setState(238); _errHandler.sync(this); _la = _input.LA(1); } @@ -1656,7 +1655,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx default: break; } - setState(243); + setState(241); match(RP); } } @@ -1703,9 +1702,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(245); + setState(243); match(ROW); - setState(246); + setState(244); fields(); } } @@ -1759,23 +1758,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(248); + setState(246); field(); - setState(253); + setState(251); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(249); + setState(247); match(COMMA); - setState(250); + setState(248); field(); } } } - setState(255); + setState(253); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); } @@ -1825,24 +1824,24 @@ public final FieldContext field() throws RecognitionException { FieldContext _localctx = new FieldContext(_ctx, getState()); enterRule(_localctx, 26, RULE_field); try { - setState(261); + setState(259); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(256); + setState(254); booleanExpression(0); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(257); + setState(255); qualifiedName(); - setState(258); + setState(256); match(ASSIGN); - setState(259); + setState(257); booleanExpression(0); } break; @@ -1902,34 +1901,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(263); + setState(261); match(FROM); - setState(264); + setState(262); fromIdentifier(); - setState(269); + setState(267); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(265); + setState(263); match(COMMA); - setState(266); + setState(264); fromIdentifier(); } } } - setState(271); + setState(269); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } - setState(273); + setState(271); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: { - setState(272); + setState(270); metadata(); } break; @@ -1979,20 +1978,20 @@ public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); enterRule(_localctx, 30, RULE_metadata); try { - setState(277); + setState(275); _errHandler.sync(this); switch (_input.LA(1)) { case METADATA: enterOuterAlt(_localctx, 1); { - setState(275); + setState(273); metadataOption(); } break; case OPENING_BRACKET: enterOuterAlt(_localctx, 2); { - setState(276); + setState(274); deprecated_metadata(); } break; @@ -2051,25 +2050,25 @@ public final MetadataOptionContext metadataOption() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(279); + setState(277); match(METADATA); - setState(280); + setState(278); fromIdentifier(); - setState(285); + setState(283); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,24,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(281); + setState(279); match(COMMA); - setState(282); + setState(280); fromIdentifier(); } } } - setState(287); + setState(285); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,24,_ctx); } @@ -2118,11 +2117,11 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition try { enterOuterAlt(_localctx, 1); { - setState(288); + setState(286); match(OPENING_BRACKET); - setState(289); + setState(287); metadataOption(); - setState(290); + setState(288); match(CLOSING_BRACKET); } } @@ -2169,9 +2168,9 @@ public final EvalCommandContext evalCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(292); + setState(290); match(EVAL); - setState(293); + setState(291); fields(); } } @@ -2224,26 +2223,26 @@ public final StatsCommandContext statsCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(295); + setState(293); match(STATS); - setState(297); + setState(295); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: { - setState(296); + setState(294); ((StatsCommandContext)_localctx).stats = fields(); } break; } - setState(301); + setState(299); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { case 1: { - setState(299); + setState(297); match(BY); - setState(300); + setState(298); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2299,18 +2298,18 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(303); + setState(301); match(INLINESTATS); - setState(304); + setState(302); ((InlinestatsCommandContext)_localctx).stats = fields(); - setState(307); + setState(305); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) { case 1: { - setState(305); + setState(303); match(BY); - setState(306); + setState(304); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -2359,7 +2358,7 @@ public final FromIdentifierContext fromIdentifier() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(309); + setState(307); _la = _input.LA(1); if ( !(_la==QUOTED_IDENTIFIER || _la==FROM_UNQUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -2421,23 +2420,23 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(311); + setState(309); identifier(); - setState(316); + setState(314); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(312); + setState(310); match(DOT); - setState(313); + setState(311); identifier(); } } } - setState(318); + setState(316); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); } @@ -2493,23 +2492,23 @@ public final QualifiedNamePatternContext qualifiedNamePattern() throws Recogniti int _alt; enterOuterAlt(_localctx, 1); { - setState(319); + setState(317); identifierPattern(); - setState(324); + setState(322); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,29,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(320); + setState(318); match(DOT); - setState(321); + setState(319); identifierPattern(); } } } - setState(326); + setState(324); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,29,_ctx); } @@ -2557,7 +2556,7 @@ public final IdentifierContext identifier() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(327); + setState(325); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -2582,12 +2581,7 @@ public final IdentifierContext identifier() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class IdentifierPatternContext extends ParserRuleContext { - public List idPattern() { - return getRuleContexts(IdPatternContext.class); - } - public IdPatternContext idPattern(int i) { - return getRuleContext(IdPatternContext.class,i); - } + public TerminalNode ID_PATTERN() { return getToken(EsqlBaseParser.ID_PATTERN, 0); } @SuppressWarnings("this-escape") public IdentifierPatternContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -2612,83 +2606,10 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); enterRule(_localctx, 50, RULE_identifierPattern); try { - int _alt; enterOuterAlt(_localctx, 1); { - setState(330); - _errHandler.sync(this); - _alt = 1; - do { - switch (_alt) { - case 1: - { - { - setState(329); - idPattern(); - } - } - break; - default: - throw new NoViableAltException(this); - } - setState(332); - _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,30,_ctx); - } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - - @SuppressWarnings("CheckReturnValue") - public static class IdPatternContext extends ParserRuleContext { - public TerminalNode UNQUOTED_ID_PATTERN() { return getToken(EsqlBaseParser.UNQUOTED_ID_PATTERN, 0); } - public TerminalNode QUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.QUOTED_IDENTIFIER, 0); } - @SuppressWarnings("this-escape") - public IdPatternContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_idPattern; } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterIdPattern(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitIdPattern(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitIdPattern(this); - else return visitor.visitChildren(this); - } - } - - public final IdPatternContext idPattern() throws RecognitionException { - IdPatternContext _localctx = new IdPatternContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_idPattern); - int _la; - try { - enterOuterAlt(_localctx, 1); - { - setState(334); - _la = _input.LA(1); - if ( !(_la==QUOTED_IDENTIFIER || _la==UNQUOTED_ID_PATTERN) ) { - _errHandler.recoverInline(this); - } - else { - if ( _input.LA(1)==Token.EOF ) matchedEOF = true; - _errHandler.reportMatch(this); - consume(); - } + setState(327); + match(ID_PATTERN); } } catch (RecognitionException re) { @@ -2953,17 +2874,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_constant); + enterRule(_localctx, 52, RULE_constant); int _la; try { - setState(378); + setState(371); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(336); + setState(329); match(NULL); } break; @@ -2971,9 +2892,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(337); + setState(330); integerValue(); - setState(338); + setState(331); match(UNQUOTED_IDENTIFIER); } break; @@ -2981,7 +2902,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(340); + setState(333); decimalValue(); } break; @@ -2989,7 +2910,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(341); + setState(334); integerValue(); } break; @@ -2997,7 +2918,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(342); + setState(335); booleanValue(); } break; @@ -3005,7 +2926,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(343); + setState(336); match(PARAM); } break; @@ -3013,7 +2934,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(344); + setState(337); string(); } break; @@ -3021,27 +2942,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(345); + setState(338); match(OPENING_BRACKET); - setState(346); + setState(339); numericValue(); - setState(351); + setState(344); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(347); + setState(340); match(COMMA); - setState(348); + setState(341); numericValue(); } } - setState(353); + setState(346); _errHandler.sync(this); _la = _input.LA(1); } - setState(354); + setState(347); match(CLOSING_BRACKET); } break; @@ -3049,27 +2970,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(356); + setState(349); match(OPENING_BRACKET); - setState(357); + setState(350); booleanValue(); - setState(362); + setState(355); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(358); + setState(351); match(COMMA); - setState(359); + setState(352); booleanValue(); } } - setState(364); + setState(357); _errHandler.sync(this); _la = _input.LA(1); } - setState(365); + setState(358); match(CLOSING_BRACKET); } break; @@ -3077,27 +2998,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(367); + setState(360); match(OPENING_BRACKET); - setState(368); + setState(361); string(); - setState(373); + setState(366); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(369); + setState(362); match(COMMA); - setState(370); + setState(363); string(); } } - setState(375); + setState(368); _errHandler.sync(this); _la = _input.LA(1); } - setState(376); + setState(369); match(CLOSING_BRACKET); } break; @@ -3140,13 +3061,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_limitCommand); + enterRule(_localctx, 54, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(380); + setState(373); match(LIMIT); - setState(381); + setState(374); match(INTEGER_LITERAL); } } @@ -3196,32 +3117,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_sortCommand); + enterRule(_localctx, 56, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(383); + setState(376); match(SORT); - setState(384); + setState(377); orderExpression(); - setState(389); + setState(382); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(385); + setState(378); match(COMMA); - setState(386); + setState(379); orderExpression(); } } } - setState(391); + setState(384); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } } } @@ -3270,19 +3191,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_orderExpression); + enterRule(_localctx, 58, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(392); + setState(385); booleanExpression(0); - setState(394); + setState(387); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) { case 1: { - setState(393); + setState(386); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -3296,14 +3217,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(398); + setState(391); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) { case 1: { - setState(396); + setState(389); match(NULLS); - setState(397); + setState(390); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -3365,32 +3286,32 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_keepCommand); + enterRule(_localctx, 60, RULE_keepCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(400); + setState(393); match(KEEP); - setState(401); + setState(394); qualifiedNamePattern(); - setState(406); + setState(399); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,38,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(402); + setState(395); match(COMMA); - setState(403); + setState(396); qualifiedNamePattern(); } } } - setState(408); + setState(401); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,38,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); } } } @@ -3440,32 +3361,32 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_dropCommand); + enterRule(_localctx, 62, RULE_dropCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(409); + setState(402); match(DROP); - setState(410); + setState(403); qualifiedNamePattern(); - setState(415); + setState(408); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,39,_ctx); + _alt = getInterpreter().adaptivePredict(_input,38,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(411); + setState(404); match(COMMA); - setState(412); + setState(405); qualifiedNamePattern(); } } } - setState(417); + setState(410); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,39,_ctx); + _alt = getInterpreter().adaptivePredict(_input,38,_ctx); } } } @@ -3515,32 +3436,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_renameCommand); + enterRule(_localctx, 64, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(418); + setState(411); match(RENAME); - setState(419); + setState(412); renameClause(); - setState(424); + setState(417); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,40,_ctx); + _alt = getInterpreter().adaptivePredict(_input,39,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(420); + setState(413); match(COMMA); - setState(421); + setState(414); renameClause(); } } } - setState(426); + setState(419); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,40,_ctx); + _alt = getInterpreter().adaptivePredict(_input,39,_ctx); } } } @@ -3588,15 +3509,15 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_renameClause); + enterRule(_localctx, 66, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(427); + setState(420); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(428); + setState(421); match(AS); - setState(429); + setState(422); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -3645,22 +3566,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_dissectCommand); + enterRule(_localctx, 68, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(431); + setState(424); match(DISSECT); - setState(432); + setState(425); primaryExpression(); - setState(433); + setState(426); string(); - setState(435); + setState(428); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) { case 1: { - setState(434); + setState(427); commandOptions(); } break; @@ -3709,15 +3630,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_grokCommand); + enterRule(_localctx, 70, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(437); + setState(430); match(GROK); - setState(438); + setState(431); primaryExpression(); - setState(439); + setState(432); string(); } } @@ -3760,13 +3681,13 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_mvExpandCommand); + enterRule(_localctx, 72, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(441); + setState(434); match(MV_EXPAND); - setState(442); + setState(435); qualifiedName(); } } @@ -3815,30 +3736,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_commandOptions); + enterRule(_localctx, 74, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(444); + setState(437); commandOption(); - setState(449); + setState(442); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,41,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(445); + setState(438); match(COMMA); - setState(446); + setState(439); commandOption(); } } } - setState(451); + setState(444); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,41,_ctx); } } } @@ -3884,15 +3805,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_commandOption); + enterRule(_localctx, 76, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(452); + setState(445); identifier(); - setState(453); + setState(446); match(ASSIGN); - setState(454); + setState(447); constant(); } } @@ -3933,12 +3854,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_booleanValue); + enterRule(_localctx, 78, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(456); + setState(449); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -3991,22 +3912,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_numericValue); + enterRule(_localctx, 80, RULE_numericValue); try { - setState(460); + setState(453); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(458); + setState(451); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(459); + setState(452); integerValue(); } break; @@ -4050,17 +3971,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_decimalValue); + enterRule(_localctx, 82, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(463); + setState(456); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(462); + setState(455); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4073,7 +3994,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(465); + setState(458); match(DECIMAL_LITERAL); } } @@ -4115,17 +4036,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_integerValue); + enterRule(_localctx, 84, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(468); + setState(461); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(467); + setState(460); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4138,7 +4059,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(470); + setState(463); match(INTEGER_LITERAL); } } @@ -4178,11 +4099,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_string); + enterRule(_localctx, 86, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(472); + setState(465); match(STRING); } } @@ -4228,12 +4149,12 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_comparisonOperator); + enterRule(_localctx, 88, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(474); + setState(467); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 285978576338026496L) != 0)) ) { _errHandler.recoverInline(this); @@ -4284,13 +4205,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_explainCommand); + enterRule(_localctx, 90, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(476); + setState(469); match(EXPLAIN); - setState(477); + setState(470); subqueryExpression(); } } @@ -4334,15 +4255,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 94, RULE_subqueryExpression); + enterRule(_localctx, 92, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(479); + setState(472); match(OPENING_BRACKET); - setState(480); + setState(473); query(0); - setState(481); + setState(474); match(CLOSING_BRACKET); } } @@ -4414,18 +4335,18 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 96, RULE_showCommand); + enterRule(_localctx, 94, RULE_showCommand); try { - setState(487); + setState(480); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) { case 1: _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(483); + setState(476); match(SHOW); - setState(484); + setState(477); match(INFO); } break; @@ -4433,9 +4354,9 @@ public final ShowCommandContext showCommand() throws RecognitionException { _localctx = new ShowFunctionsContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(485); + setState(478); match(SHOW); - setState(486); + setState(479); match(FUNCTIONS); } break; @@ -4495,53 +4416,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 98, RULE_enrichCommand); + enterRule(_localctx, 96, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(489); + setState(482); match(ENRICH); - setState(490); + setState(483); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(493); + setState(486); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: { - setState(491); + setState(484); match(ON); - setState(492); + setState(485); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(504); + setState(497); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(495); + setState(488); match(WITH); - setState(496); + setState(489); enrichWithClause(); - setState(501); + setState(494); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(497); + setState(490); match(COMMA); - setState(498); + setState(491); enrichWithClause(); } } } - setState(503); + setState(496); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); } } break; @@ -4592,23 +4513,23 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 100, RULE_enrichWithClause); + enterRule(_localctx, 98, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(509); + setState(502); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { case 1: { - setState(506); + setState(499); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(507); + setState(500); match(ASSIGN); } break; } - setState(511); + setState(504); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -4661,7 +4582,7 @@ private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, } public static final String _serializedATN = - "\u0004\u0001h\u0202\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ + "\u0004\u0001h\u01fb\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ @@ -4674,321 +4595,317 @@ private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"+ "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+ "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ - "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+ - "2\u00072\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001p\b\u0001\n\u0001"+ - "\f\u0001s\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003"+ - "\u0002y\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0001"+ + "\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0005\u0001n\b\u0001\n\u0001\f\u0001q\t"+ + "\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002w\b"+ + "\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u0088\b\u0003\u0001\u0004\u0001"+ - "\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u0094\b\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u009b\b\u0005\n"+ - "\u0005\f\u0005\u009e\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0003\u0005\u00a5\b\u0005\u0001\u0005\u0001\u0005\u0003"+ - "\u0005\u00a9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0005\u0005\u00b1\b\u0005\n\u0005\f\u0005\u00b4\t\u0005"+ - "\u0001\u0006\u0001\u0006\u0003\u0006\u00b8\b\u0006\u0001\u0006\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00bf\b\u0006\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0003\u0006\u00c4\b\u0006\u0001\u0007\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00cb\b\u0007\u0001\b"+ - "\u0001\b\u0001\b\u0001\b\u0003\b\u00d1\b\b\u0001\b\u0001\b\u0001\b\u0001"+ - "\b\u0001\b\u0001\b\u0005\b\u00d9\b\b\n\b\f\b\u00dc\t\b\u0001\t\u0001\t"+ - "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00e5\b\t\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0005\n\u00ed\b\n\n\n\f\n\u00f0\t\n"+ - "\u0003\n\u00f2\b\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\f\u0001\f\u0001\f\u0005\f\u00fc\b\f\n\f\f\f\u00ff\t\f\u0001\r\u0001"+ - "\r\u0001\r\u0001\r\u0001\r\u0003\r\u0106\b\r\u0001\u000e\u0001\u000e\u0001"+ - "\u000e\u0001\u000e\u0005\u000e\u010c\b\u000e\n\u000e\f\u000e\u010f\t\u000e"+ - "\u0001\u000e\u0003\u000e\u0112\b\u000e\u0001\u000f\u0001\u000f\u0003\u000f"+ - "\u0116\b\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005\u0010"+ - "\u011c\b\u0010\n\u0010\f\u0010\u011f\t\u0010\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001"+ - "\u0013\u0003\u0013\u012a\b\u0013\u0001\u0013\u0001\u0013\u0003\u0013\u012e"+ - "\b\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u0134"+ + "\u0003\u0001\u0003\u0003\u0003\u0086\b\u0003\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0003\u0005\u0092\b\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u0099\b\u0005\n\u0005\f\u0005"+ + "\u009c\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0003\u0005\u00a3\b\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00a7\b"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0005\u0005\u00af\b\u0005\n\u0005\f\u0005\u00b2\t\u0005\u0001\u0006"+ + "\u0001\u0006\u0003\u0006\u00b6\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ + "\u0001\u0006\u0001\u0006\u0003\u0006\u00bd\b\u0006\u0001\u0006\u0001\u0006"+ + "\u0001\u0006\u0003\u0006\u00c2\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007"+ + "\u0001\u0007\u0001\u0007\u0003\u0007\u00c9\b\u0007\u0001\b\u0001\b\u0001"+ + "\b\u0001\b\u0003\b\u00cf\b\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ + "\b\u0005\b\u00d7\b\b\n\b\f\b\u00da\t\b\u0001\t\u0001\t\u0001\t\u0001\t"+ + "\u0001\t\u0001\t\u0001\t\u0003\t\u00e3\b\t\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\n\u0005\n\u00eb\b\n\n\n\f\n\u00ee\t\n\u0003\n\u00f0\b"+ + "\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f"+ + "\u0001\f\u0005\f\u00fa\b\f\n\f\f\f\u00fd\t\f\u0001\r\u0001\r\u0001\r\u0001"+ + "\r\u0001\r\u0003\r\u0104\b\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+ + "\u000e\u0005\u000e\u010a\b\u000e\n\u000e\f\u000e\u010d\t\u000e\u0001\u000e"+ + "\u0003\u000e\u0110\b\u000e\u0001\u000f\u0001\u000f\u0003\u000f\u0114\b"+ + "\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005\u0010\u011a"+ + "\b\u0010\n\u0010\f\u0010\u011d\t\u0010\u0001\u0011\u0001\u0011\u0001\u0011"+ + "\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ + "\u0003\u0013\u0128\b\u0013\u0001\u0013\u0001\u0013\u0003\u0013\u012c\b"+ + "\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u0132"+ "\b\u0014\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0005"+ - "\u0016\u013b\b\u0016\n\u0016\f\u0016\u013e\t\u0016\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0005\u0017\u0143\b\u0017\n\u0017\f\u0017\u0146\t\u0017\u0001"+ - "\u0018\u0001\u0018\u0001\u0019\u0004\u0019\u014b\b\u0019\u000b\u0019\f"+ - "\u0019\u014c\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+ - "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"+ - "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u015e\b\u001b"+ - "\n\u001b\f\u001b\u0161\t\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001"+ - "\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u0169\b\u001b\n\u001b\f\u001b"+ - "\u016c\t\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"+ - "\u0001\u001b\u0005\u001b\u0174\b\u001b\n\u001b\f\u001b\u0177\t\u001b\u0001"+ - "\u001b\u0001\u001b\u0003\u001b\u017b\b\u001b\u0001\u001c\u0001\u001c\u0001"+ - "\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u0184"+ - "\b\u001d\n\u001d\f\u001d\u0187\t\u001d\u0001\u001e\u0001\u001e\u0003\u001e"+ - "\u018b\b\u001e\u0001\u001e\u0001\u001e\u0003\u001e\u018f\b\u001e\u0001"+ - "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u0195\b\u001f\n"+ - "\u001f\f\u001f\u0198\t\u001f\u0001 \u0001 \u0001 \u0001 \u0005 \u019e"+ - "\b \n \f \u01a1\t \u0001!\u0001!\u0001!\u0001!\u0005!\u01a7\b!\n!\f!\u01aa"+ - "\t!\u0001\"\u0001\"\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0001#\u0003"+ - "#\u01b4\b#\u0001$\u0001$\u0001$\u0001$\u0001%\u0001%\u0001%\u0001&\u0001"+ - "&\u0001&\u0005&\u01c0\b&\n&\f&\u01c3\t&\u0001\'\u0001\'\u0001\'\u0001"+ - "\'\u0001(\u0001(\u0001)\u0001)\u0003)\u01cd\b)\u0001*\u0003*\u01d0\b*"+ - "\u0001*\u0001*\u0001+\u0003+\u01d5\b+\u0001+\u0001+\u0001,\u0001,\u0001"+ - "-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0001/\u00010\u0001"+ - "0\u00010\u00010\u00030\u01e8\b0\u00011\u00011\u00011\u00011\u00031\u01ee"+ - "\b1\u00011\u00011\u00011\u00011\u00051\u01f4\b1\n1\f1\u01f7\t1\u00031"+ - "\u01f9\b1\u00012\u00012\u00012\u00032\u01fe\b2\u00012\u00012\u00012\u0000"+ - "\u0003\u0002\n\u00103\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012"+ - "\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\"+ - "^`bd\u0000\t\u0001\u0000:;\u0001\u0000<>\u0002\u0000BBGG\u0001\u0000A"+ - "B\u0002\u0000BBKK\u0002\u0000\u001f\u001f\"\"\u0001\u0000%&\u0002\u0000"+ - "$$22\u0001\u000039\u021c\u0000f\u0001\u0000\u0000\u0000\u0002i\u0001\u0000"+ - "\u0000\u0000\u0004x\u0001\u0000\u0000\u0000\u0006\u0087\u0001\u0000\u0000"+ - "\u0000\b\u0089\u0001\u0000\u0000\u0000\n\u00a8\u0001\u0000\u0000\u0000"+ - "\f\u00c3\u0001\u0000\u0000\u0000\u000e\u00ca\u0001\u0000\u0000\u0000\u0010"+ - "\u00d0\u0001\u0000\u0000\u0000\u0012\u00e4\u0001\u0000\u0000\u0000\u0014"+ - "\u00e6\u0001\u0000\u0000\u0000\u0016\u00f5\u0001\u0000\u0000\u0000\u0018"+ - "\u00f8\u0001\u0000\u0000\u0000\u001a\u0105\u0001\u0000\u0000\u0000\u001c"+ - "\u0107\u0001\u0000\u0000\u0000\u001e\u0115\u0001\u0000\u0000\u0000 \u0117"+ - "\u0001\u0000\u0000\u0000\"\u0120\u0001\u0000\u0000\u0000$\u0124\u0001"+ - "\u0000\u0000\u0000&\u0127\u0001\u0000\u0000\u0000(\u012f\u0001\u0000\u0000"+ - "\u0000*\u0135\u0001\u0000\u0000\u0000,\u0137\u0001\u0000\u0000\u0000."+ - "\u013f\u0001\u0000\u0000\u00000\u0147\u0001\u0000\u0000\u00002\u014a\u0001"+ - "\u0000\u0000\u00004\u014e\u0001\u0000\u0000\u00006\u017a\u0001\u0000\u0000"+ - "\u00008\u017c\u0001\u0000\u0000\u0000:\u017f\u0001\u0000\u0000\u0000<"+ - "\u0188\u0001\u0000\u0000\u0000>\u0190\u0001\u0000\u0000\u0000@\u0199\u0001"+ - "\u0000\u0000\u0000B\u01a2\u0001\u0000\u0000\u0000D\u01ab\u0001\u0000\u0000"+ - "\u0000F\u01af\u0001\u0000\u0000\u0000H\u01b5\u0001\u0000\u0000\u0000J"+ - "\u01b9\u0001\u0000\u0000\u0000L\u01bc\u0001\u0000\u0000\u0000N\u01c4\u0001"+ - "\u0000\u0000\u0000P\u01c8\u0001\u0000\u0000\u0000R\u01cc\u0001\u0000\u0000"+ - "\u0000T\u01cf\u0001\u0000\u0000\u0000V\u01d4\u0001\u0000\u0000\u0000X"+ - "\u01d8\u0001\u0000\u0000\u0000Z\u01da\u0001\u0000\u0000\u0000\\\u01dc"+ - "\u0001\u0000\u0000\u0000^\u01df\u0001\u0000\u0000\u0000`\u01e7\u0001\u0000"+ - "\u0000\u0000b\u01e9\u0001\u0000\u0000\u0000d\u01fd\u0001\u0000\u0000\u0000"+ - "fg\u0003\u0002\u0001\u0000gh\u0005\u0000\u0000\u0001h\u0001\u0001\u0000"+ - "\u0000\u0000ij\u0006\u0001\uffff\uffff\u0000jk\u0003\u0004\u0002\u0000"+ - "kq\u0001\u0000\u0000\u0000lm\n\u0001\u0000\u0000mn\u0005\u0019\u0000\u0000"+ - "np\u0003\u0006\u0003\u0000ol\u0001\u0000\u0000\u0000ps\u0001\u0000\u0000"+ - "\u0000qo\u0001\u0000\u0000\u0000qr\u0001\u0000\u0000\u0000r\u0003\u0001"+ - "\u0000\u0000\u0000sq\u0001\u0000\u0000\u0000ty\u0003\\.\u0000uy\u0003"+ - "\u001c\u000e\u0000vy\u0003\u0016\u000b\u0000wy\u0003`0\u0000xt\u0001\u0000"+ - "\u0000\u0000xu\u0001\u0000\u0000\u0000xv\u0001\u0000\u0000\u0000xw\u0001"+ - "\u0000\u0000\u0000y\u0005\u0001\u0000\u0000\u0000z\u0088\u0003$\u0012"+ - "\u0000{\u0088\u0003(\u0014\u0000|\u0088\u00038\u001c\u0000}\u0088\u0003"+ - ">\u001f\u0000~\u0088\u0003:\u001d\u0000\u007f\u0088\u0003&\u0013\u0000"+ - "\u0080\u0088\u0003\b\u0004\u0000\u0081\u0088\u0003@ \u0000\u0082\u0088"+ - "\u0003B!\u0000\u0083\u0088\u0003F#\u0000\u0084\u0088\u0003H$\u0000\u0085"+ - "\u0088\u0003b1\u0000\u0086\u0088\u0003J%\u0000\u0087z\u0001\u0000\u0000"+ - "\u0000\u0087{\u0001\u0000\u0000\u0000\u0087|\u0001\u0000\u0000\u0000\u0087"+ - "}\u0001\u0000\u0000\u0000\u0087~\u0001\u0000\u0000\u0000\u0087\u007f\u0001"+ - "\u0000\u0000\u0000\u0087\u0080\u0001\u0000\u0000\u0000\u0087\u0081\u0001"+ - "\u0000\u0000\u0000\u0087\u0082\u0001\u0000\u0000\u0000\u0087\u0083\u0001"+ - "\u0000\u0000\u0000\u0087\u0084\u0001\u0000\u0000\u0000\u0087\u0085\u0001"+ - "\u0000\u0000\u0000\u0087\u0086\u0001\u0000\u0000\u0000\u0088\u0007\u0001"+ - "\u0000\u0000\u0000\u0089\u008a\u0005\u0011\u0000\u0000\u008a\u008b\u0003"+ - "\n\u0005\u0000\u008b\t\u0001\u0000\u0000\u0000\u008c\u008d\u0006\u0005"+ - "\uffff\uffff\u0000\u008d\u008e\u0005+\u0000\u0000\u008e\u00a9\u0003\n"+ - "\u0005\u0007\u008f\u00a9\u0003\u000e\u0007\u0000\u0090\u00a9\u0003\f\u0006"+ - "\u0000\u0091\u0093\u0003\u000e\u0007\u0000\u0092\u0094\u0005+\u0000\u0000"+ - "\u0093\u0092\u0001\u0000\u0000\u0000\u0093\u0094\u0001\u0000\u0000\u0000"+ - "\u0094\u0095\u0001\u0000\u0000\u0000\u0095\u0096\u0005(\u0000\u0000\u0096"+ - "\u0097\u0005\'\u0000\u0000\u0097\u009c\u0003\u000e\u0007\u0000\u0098\u0099"+ - "\u0005!\u0000\u0000\u0099\u009b\u0003\u000e\u0007\u0000\u009a\u0098\u0001"+ - "\u0000\u0000\u0000\u009b\u009e\u0001\u0000\u0000\u0000\u009c\u009a\u0001"+ - "\u0000\u0000\u0000\u009c\u009d\u0001\u0000\u0000\u0000\u009d\u009f\u0001"+ - "\u0000\u0000\u0000\u009e\u009c\u0001\u0000\u0000\u0000\u009f\u00a0\u0005"+ - "1\u0000\u0000\u00a0\u00a9\u0001\u0000\u0000\u0000\u00a1\u00a2\u0003\u000e"+ - "\u0007\u0000\u00a2\u00a4\u0005)\u0000\u0000\u00a3\u00a5\u0005+\u0000\u0000"+ - "\u00a4\u00a3\u0001\u0000\u0000\u0000\u00a4\u00a5\u0001\u0000\u0000\u0000"+ - "\u00a5\u00a6\u0001\u0000\u0000\u0000\u00a6\u00a7\u0005,\u0000\u0000\u00a7"+ - "\u00a9\u0001\u0000\u0000\u0000\u00a8\u008c\u0001\u0000\u0000\u0000\u00a8"+ - "\u008f\u0001\u0000\u0000\u0000\u00a8\u0090\u0001\u0000\u0000\u0000\u00a8"+ - "\u0091\u0001\u0000\u0000\u0000\u00a8\u00a1\u0001\u0000\u0000\u0000\u00a9"+ - "\u00b2\u0001\u0000\u0000\u0000\u00aa\u00ab\n\u0004\u0000\u0000\u00ab\u00ac"+ - "\u0005\u001e\u0000\u0000\u00ac\u00b1\u0003\n\u0005\u0005\u00ad\u00ae\n"+ - "\u0003\u0000\u0000\u00ae\u00af\u0005.\u0000\u0000\u00af\u00b1\u0003\n"+ - "\u0005\u0004\u00b0\u00aa\u0001\u0000\u0000\u0000\u00b0\u00ad\u0001\u0000"+ - "\u0000\u0000\u00b1\u00b4\u0001\u0000\u0000\u0000\u00b2\u00b0\u0001\u0000"+ - "\u0000\u0000\u00b2\u00b3\u0001\u0000\u0000\u0000\u00b3\u000b\u0001\u0000"+ - "\u0000\u0000\u00b4\u00b2\u0001\u0000\u0000\u0000\u00b5\u00b7\u0003\u000e"+ - "\u0007\u0000\u00b6\u00b8\u0005+\u0000\u0000\u00b7\u00b6\u0001\u0000\u0000"+ - "\u0000\u00b7\u00b8\u0001\u0000\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000"+ - "\u0000\u00b9\u00ba\u0005*\u0000\u0000\u00ba\u00bb\u0003X,\u0000\u00bb"+ - "\u00c4\u0001\u0000\u0000\u0000\u00bc\u00be\u0003\u000e\u0007\u0000\u00bd"+ - "\u00bf\u0005+\u0000\u0000\u00be\u00bd\u0001\u0000\u0000\u0000\u00be\u00bf"+ - "\u0001\u0000\u0000\u0000\u00bf\u00c0\u0001\u0000\u0000\u0000\u00c0\u00c1"+ - "\u00050\u0000\u0000\u00c1\u00c2\u0003X,\u0000\u00c2\u00c4\u0001\u0000"+ - "\u0000\u0000\u00c3\u00b5\u0001\u0000\u0000\u0000\u00c3\u00bc\u0001\u0000"+ - "\u0000\u0000\u00c4\r\u0001\u0000\u0000\u0000\u00c5\u00cb\u0003\u0010\b"+ - "\u0000\u00c6\u00c7\u0003\u0010\b\u0000\u00c7\u00c8\u0003Z-\u0000\u00c8"+ - "\u00c9\u0003\u0010\b\u0000\u00c9\u00cb\u0001\u0000\u0000\u0000\u00ca\u00c5"+ - "\u0001\u0000\u0000\u0000\u00ca\u00c6\u0001\u0000\u0000\u0000\u00cb\u000f"+ - "\u0001\u0000\u0000\u0000\u00cc\u00cd\u0006\b\uffff\uffff\u0000\u00cd\u00d1"+ - "\u0003\u0012\t\u0000\u00ce\u00cf\u0007\u0000\u0000\u0000\u00cf\u00d1\u0003"+ - "\u0010\b\u0003\u00d0\u00cc\u0001\u0000\u0000\u0000\u00d0\u00ce\u0001\u0000"+ - "\u0000\u0000\u00d1\u00da\u0001\u0000\u0000\u0000\u00d2\u00d3\n\u0002\u0000"+ - "\u0000\u00d3\u00d4\u0007\u0001\u0000\u0000\u00d4\u00d9\u0003\u0010\b\u0003"+ - "\u00d5\u00d6\n\u0001\u0000\u0000\u00d6\u00d7\u0007\u0000\u0000\u0000\u00d7"+ - "\u00d9\u0003\u0010\b\u0002\u00d8\u00d2\u0001\u0000\u0000\u0000\u00d8\u00d5"+ - "\u0001\u0000\u0000\u0000\u00d9\u00dc\u0001\u0000\u0000\u0000\u00da\u00d8"+ - "\u0001\u0000\u0000\u0000\u00da\u00db\u0001\u0000\u0000\u0000\u00db\u0011"+ - "\u0001\u0000\u0000\u0000\u00dc\u00da\u0001\u0000\u0000\u0000\u00dd\u00e5"+ - "\u00036\u001b\u0000\u00de\u00e5\u0003,\u0016\u0000\u00df\u00e5\u0003\u0014"+ - "\n\u0000\u00e0\u00e1\u0005\'\u0000\u0000\u00e1\u00e2\u0003\n\u0005\u0000"+ - "\u00e2\u00e3\u00051\u0000\u0000\u00e3\u00e5\u0001\u0000\u0000\u0000\u00e4"+ - "\u00dd\u0001\u0000\u0000\u0000\u00e4\u00de\u0001\u0000\u0000\u0000\u00e4"+ - "\u00df\u0001\u0000\u0000\u0000\u00e4\u00e0\u0001\u0000\u0000\u0000\u00e5"+ - "\u0013\u0001\u0000\u0000\u0000\u00e6\u00e7\u00030\u0018\u0000\u00e7\u00f1"+ - "\u0005\'\u0000\u0000\u00e8\u00f2\u0005<\u0000\u0000\u00e9\u00ee\u0003"+ - "\n\u0005\u0000\u00ea\u00eb\u0005!\u0000\u0000\u00eb\u00ed\u0003\n\u0005"+ - "\u0000\u00ec\u00ea\u0001\u0000\u0000\u0000\u00ed\u00f0\u0001\u0000\u0000"+ - "\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001\u0000\u0000"+ - "\u0000\u00ef\u00f2\u0001\u0000\u0000\u0000\u00f0\u00ee\u0001\u0000\u0000"+ - "\u0000\u00f1\u00e8\u0001\u0000\u0000\u0000\u00f1\u00e9\u0001\u0000\u0000"+ - "\u0000\u00f1\u00f2\u0001\u0000\u0000\u0000\u00f2\u00f3\u0001\u0000\u0000"+ - "\u0000\u00f3\u00f4\u00051\u0000\u0000\u00f4\u0015\u0001\u0000\u0000\u0000"+ - "\u00f5\u00f6\u0005\r\u0000\u0000\u00f6\u00f7\u0003\u0018\f\u0000\u00f7"+ - "\u0017\u0001\u0000\u0000\u0000\u00f8\u00fd\u0003\u001a\r\u0000\u00f9\u00fa"+ - "\u0005!\u0000\u0000\u00fa\u00fc\u0003\u001a\r\u0000\u00fb\u00f9\u0001"+ - "\u0000\u0000\u0000\u00fc\u00ff\u0001\u0000\u0000\u0000\u00fd\u00fb\u0001"+ - "\u0000\u0000\u0000\u00fd\u00fe\u0001\u0000\u0000\u0000\u00fe\u0019\u0001"+ - "\u0000\u0000\u0000\u00ff\u00fd\u0001\u0000\u0000\u0000\u0100\u0106\u0003"+ - "\n\u0005\u0000\u0101\u0102\u0003,\u0016\u0000\u0102\u0103\u0005 \u0000"+ - "\u0000\u0103\u0104\u0003\n\u0005\u0000\u0104\u0106\u0001\u0000\u0000\u0000"+ - "\u0105\u0100\u0001\u0000\u0000\u0000\u0105\u0101\u0001\u0000\u0000\u0000"+ - "\u0106\u001b\u0001\u0000\u0000\u0000\u0107\u0108\u0005\u0006\u0000\u0000"+ - "\u0108\u010d\u0003*\u0015\u0000\u0109\u010a\u0005!\u0000\u0000\u010a\u010c"+ - "\u0003*\u0015\u0000\u010b\u0109\u0001\u0000\u0000\u0000\u010c\u010f\u0001"+ - "\u0000\u0000\u0000\u010d\u010b\u0001\u0000\u0000\u0000\u010d\u010e\u0001"+ - "\u0000\u0000\u0000\u010e\u0111\u0001\u0000\u0000\u0000\u010f\u010d\u0001"+ - "\u0000\u0000\u0000\u0110\u0112\u0003\u001e\u000f\u0000\u0111\u0110\u0001"+ - "\u0000\u0000\u0000\u0111\u0112\u0001\u0000\u0000\u0000\u0112\u001d\u0001"+ - "\u0000\u0000\u0000\u0113\u0116\u0003 \u0010\u0000\u0114\u0116\u0003\""+ - "\u0011\u0000\u0115\u0113\u0001\u0000\u0000\u0000\u0115\u0114\u0001\u0000"+ - "\u0000\u0000\u0116\u001f\u0001\u0000\u0000\u0000\u0117\u0118\u0005F\u0000"+ - "\u0000\u0118\u011d\u0003*\u0015\u0000\u0119\u011a\u0005!\u0000\u0000\u011a"+ - "\u011c\u0003*\u0015\u0000\u011b\u0119\u0001\u0000\u0000\u0000\u011c\u011f"+ - "\u0001\u0000\u0000\u0000\u011d\u011b\u0001\u0000\u0000\u0000\u011d\u011e"+ - "\u0001\u0000\u0000\u0000\u011e!\u0001\u0000\u0000\u0000\u011f\u011d\u0001"+ - "\u0000\u0000\u0000\u0120\u0121\u0005?\u0000\u0000\u0121\u0122\u0003 \u0010"+ - "\u0000\u0122\u0123\u0005@\u0000\u0000\u0123#\u0001\u0000\u0000\u0000\u0124"+ - "\u0125\u0005\u0004\u0000\u0000\u0125\u0126\u0003\u0018\f\u0000\u0126%"+ - "\u0001\u0000\u0000\u0000\u0127\u0129\u0005\u0010\u0000\u0000\u0128\u012a"+ - "\u0003\u0018\f\u0000\u0129\u0128\u0001\u0000\u0000\u0000\u0129\u012a\u0001"+ - "\u0000\u0000\u0000\u012a\u012d\u0001\u0000\u0000\u0000\u012b\u012c\u0005"+ - "\u001d\u0000\u0000\u012c\u012e\u0003\u0018\f\u0000\u012d\u012b\u0001\u0000"+ - "\u0000\u0000\u012d\u012e\u0001\u0000\u0000\u0000\u012e\'\u0001\u0000\u0000"+ - "\u0000\u012f\u0130\u0005\b\u0000\u0000\u0130\u0133\u0003\u0018\f\u0000"+ - "\u0131\u0132\u0005\u001d\u0000\u0000\u0132\u0134\u0003\u0018\f\u0000\u0133"+ - "\u0131\u0001\u0000\u0000\u0000\u0133\u0134\u0001\u0000\u0000\u0000\u0134"+ - ")\u0001\u0000\u0000\u0000\u0135\u0136\u0007\u0002\u0000\u0000\u0136+\u0001"+ - "\u0000\u0000\u0000\u0137\u013c\u00030\u0018\u0000\u0138\u0139\u0005#\u0000"+ - "\u0000\u0139\u013b\u00030\u0018\u0000\u013a\u0138\u0001\u0000\u0000\u0000"+ - "\u013b\u013e\u0001\u0000\u0000\u0000\u013c\u013a\u0001\u0000\u0000\u0000"+ - "\u013c\u013d\u0001\u0000\u0000\u0000\u013d-\u0001\u0000\u0000\u0000\u013e"+ - "\u013c\u0001\u0000\u0000\u0000\u013f\u0144\u00032\u0019\u0000\u0140\u0141"+ - "\u0005#\u0000\u0000\u0141\u0143\u00032\u0019\u0000\u0142\u0140\u0001\u0000"+ - "\u0000\u0000\u0143\u0146\u0001\u0000\u0000\u0000\u0144\u0142\u0001\u0000"+ - "\u0000\u0000\u0144\u0145\u0001\u0000\u0000\u0000\u0145/\u0001\u0000\u0000"+ - "\u0000\u0146\u0144\u0001\u0000\u0000\u0000\u0147\u0148\u0007\u0003\u0000"+ - "\u0000\u01481\u0001\u0000\u0000\u0000\u0149\u014b\u00034\u001a\u0000\u014a"+ - "\u0149\u0001\u0000\u0000\u0000\u014b\u014c\u0001\u0000\u0000\u0000\u014c"+ - "\u014a\u0001\u0000\u0000\u0000\u014c\u014d\u0001\u0000\u0000\u0000\u014d"+ - "3\u0001\u0000\u0000\u0000\u014e\u014f\u0007\u0004\u0000\u0000\u014f5\u0001"+ - "\u0000\u0000\u0000\u0150\u017b\u0005,\u0000\u0000\u0151\u0152\u0003V+"+ - "\u0000\u0152\u0153\u0005A\u0000\u0000\u0153\u017b\u0001\u0000\u0000\u0000"+ - "\u0154\u017b\u0003T*\u0000\u0155\u017b\u0003V+\u0000\u0156\u017b\u0003"+ - "P(\u0000\u0157\u017b\u0005/\u0000\u0000\u0158\u017b\u0003X,\u0000\u0159"+ - "\u015a\u0005?\u0000\u0000\u015a\u015f\u0003R)\u0000\u015b\u015c\u0005"+ - "!\u0000\u0000\u015c\u015e\u0003R)\u0000\u015d\u015b\u0001\u0000\u0000"+ - "\u0000\u015e\u0161\u0001\u0000\u0000\u0000\u015f\u015d\u0001\u0000\u0000"+ - "\u0000\u015f\u0160\u0001\u0000\u0000\u0000\u0160\u0162\u0001\u0000\u0000"+ - "\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0163\u0005@\u0000\u0000"+ - "\u0163\u017b\u0001\u0000\u0000\u0000\u0164\u0165\u0005?\u0000\u0000\u0165"+ - "\u016a\u0003P(\u0000\u0166\u0167\u0005!\u0000\u0000\u0167\u0169\u0003"+ - "P(\u0000\u0168\u0166\u0001\u0000\u0000\u0000\u0169\u016c\u0001\u0000\u0000"+ - "\u0000\u016a\u0168\u0001\u0000\u0000\u0000\u016a\u016b\u0001\u0000\u0000"+ - "\u0000\u016b\u016d\u0001\u0000\u0000\u0000\u016c\u016a\u0001\u0000\u0000"+ - "\u0000\u016d\u016e\u0005@\u0000\u0000\u016e\u017b\u0001\u0000\u0000\u0000"+ - "\u016f\u0170\u0005?\u0000\u0000\u0170\u0175\u0003X,\u0000\u0171\u0172"+ - "\u0005!\u0000\u0000\u0172\u0174\u0003X,\u0000\u0173\u0171\u0001\u0000"+ - "\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000\u0175\u0173\u0001\u0000"+ - "\u0000\u0000\u0175\u0176\u0001\u0000\u0000\u0000\u0176\u0178\u0001\u0000"+ - "\u0000\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178\u0179\u0005@\u0000"+ - "\u0000\u0179\u017b\u0001\u0000\u0000\u0000\u017a\u0150\u0001\u0000\u0000"+ - "\u0000\u017a\u0151\u0001\u0000\u0000\u0000\u017a\u0154\u0001\u0000\u0000"+ - "\u0000\u017a\u0155\u0001\u0000\u0000\u0000\u017a\u0156\u0001\u0000\u0000"+ - "\u0000\u017a\u0157\u0001\u0000\u0000\u0000\u017a\u0158\u0001\u0000\u0000"+ - "\u0000\u017a\u0159\u0001\u0000\u0000\u0000\u017a\u0164\u0001\u0000\u0000"+ - "\u0000\u017a\u016f\u0001\u0000\u0000\u0000\u017b7\u0001\u0000\u0000\u0000"+ - "\u017c\u017d\u0005\n\u0000\u0000\u017d\u017e\u0005\u001b\u0000\u0000\u017e"+ - "9\u0001\u0000\u0000\u0000\u017f\u0180\u0005\u000f\u0000\u0000\u0180\u0185"+ - "\u0003<\u001e\u0000\u0181\u0182\u0005!\u0000\u0000\u0182\u0184\u0003<"+ - "\u001e\u0000\u0183\u0181\u0001\u0000\u0000\u0000\u0184\u0187\u0001\u0000"+ - "\u0000\u0000\u0185\u0183\u0001\u0000\u0000\u0000\u0185\u0186\u0001\u0000"+ - "\u0000\u0000\u0186;\u0001\u0000\u0000\u0000\u0187\u0185\u0001\u0000\u0000"+ - "\u0000\u0188\u018a\u0003\n\u0005\u0000\u0189\u018b\u0007\u0005\u0000\u0000"+ - "\u018a\u0189\u0001\u0000\u0000\u0000\u018a\u018b\u0001\u0000\u0000\u0000"+ - "\u018b\u018e\u0001\u0000\u0000\u0000\u018c\u018d\u0005-\u0000\u0000\u018d"+ - "\u018f\u0007\u0006\u0000\u0000\u018e\u018c\u0001\u0000\u0000\u0000\u018e"+ - "\u018f\u0001\u0000\u0000\u0000\u018f=\u0001\u0000\u0000\u0000\u0190\u0191"+ - "\u0005\t\u0000\u0000\u0191\u0196\u0003.\u0017\u0000\u0192\u0193\u0005"+ - "!\u0000\u0000\u0193\u0195\u0003.\u0017\u0000\u0194\u0192\u0001\u0000\u0000"+ - "\u0000\u0195\u0198\u0001\u0000\u0000\u0000\u0196\u0194\u0001\u0000\u0000"+ - "\u0000\u0196\u0197\u0001\u0000\u0000\u0000\u0197?\u0001\u0000\u0000\u0000"+ - "\u0198\u0196\u0001\u0000\u0000\u0000\u0199\u019a\u0005\u0002\u0000\u0000"+ - "\u019a\u019f\u0003.\u0017\u0000\u019b\u019c\u0005!\u0000\u0000\u019c\u019e"+ - "\u0003.\u0017\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e\u01a1\u0001"+ - "\u0000\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u019f\u01a0\u0001"+ - "\u0000\u0000\u0000\u01a0A\u0001\u0000\u0000\u0000\u01a1\u019f\u0001\u0000"+ - "\u0000\u0000\u01a2\u01a3\u0005\f\u0000\u0000\u01a3\u01a8\u0003D\"\u0000"+ - "\u01a4\u01a5\u0005!\u0000\u0000\u01a5\u01a7\u0003D\"\u0000\u01a6\u01a4"+ - "\u0001\u0000\u0000\u0000\u01a7\u01aa\u0001\u0000\u0000\u0000\u01a8\u01a6"+ - "\u0001\u0000\u0000\u0000\u01a8\u01a9\u0001\u0000\u0000\u0000\u01a9C\u0001"+ - "\u0000\u0000\u0000\u01aa\u01a8\u0001\u0000\u0000\u0000\u01ab\u01ac\u0003"+ - ".\u0017\u0000\u01ac\u01ad\u0005O\u0000\u0000\u01ad\u01ae\u0003.\u0017"+ - "\u0000\u01aeE\u0001\u0000\u0000\u0000\u01af\u01b0\u0005\u0001\u0000\u0000"+ - "\u01b0\u01b1\u0003\u0012\t\u0000\u01b1\u01b3\u0003X,\u0000\u01b2\u01b4"+ - "\u0003L&\u0000\u01b3\u01b2\u0001\u0000\u0000\u0000\u01b3\u01b4\u0001\u0000"+ - "\u0000\u0000\u01b4G\u0001\u0000\u0000\u0000\u01b5\u01b6\u0005\u0007\u0000"+ - "\u0000\u01b6\u01b7\u0003\u0012\t\u0000\u01b7\u01b8\u0003X,\u0000\u01b8"+ - "I\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005\u000b\u0000\u0000\u01ba\u01bb"+ - "\u0003,\u0016\u0000\u01bbK\u0001\u0000\u0000\u0000\u01bc\u01c1\u0003N"+ - "\'\u0000\u01bd\u01be\u0005!\u0000\u0000\u01be\u01c0\u0003N\'\u0000\u01bf"+ - "\u01bd\u0001\u0000\u0000\u0000\u01c0\u01c3\u0001\u0000\u0000\u0000\u01c1"+ - "\u01bf\u0001\u0000\u0000\u0000\u01c1\u01c2\u0001\u0000\u0000\u0000\u01c2"+ - "M\u0001\u0000\u0000\u0000\u01c3\u01c1\u0001\u0000\u0000\u0000\u01c4\u01c5"+ - "\u00030\u0018\u0000\u01c5\u01c6\u0005 \u0000\u0000\u01c6\u01c7\u00036"+ - "\u001b\u0000\u01c7O\u0001\u0000\u0000\u0000\u01c8\u01c9\u0007\u0007\u0000"+ - "\u0000\u01c9Q\u0001\u0000\u0000\u0000\u01ca\u01cd\u0003T*\u0000\u01cb"+ - "\u01cd\u0003V+\u0000\u01cc\u01ca\u0001\u0000\u0000\u0000\u01cc\u01cb\u0001"+ - "\u0000\u0000\u0000\u01cdS\u0001\u0000\u0000\u0000\u01ce\u01d0\u0007\u0000"+ - "\u0000\u0000\u01cf\u01ce\u0001\u0000\u0000\u0000\u01cf\u01d0\u0001\u0000"+ - "\u0000\u0000\u01d0\u01d1\u0001\u0000\u0000\u0000\u01d1\u01d2\u0005\u001c"+ - "\u0000\u0000\u01d2U\u0001\u0000\u0000\u0000\u01d3\u01d5\u0007\u0000\u0000"+ - "\u0000\u01d4\u01d3\u0001\u0000\u0000\u0000\u01d4\u01d5\u0001\u0000\u0000"+ - "\u0000\u01d5\u01d6\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005\u001b\u0000"+ - "\u0000\u01d7W\u0001\u0000\u0000\u0000\u01d8\u01d9\u0005\u001a\u0000\u0000"+ - "\u01d9Y\u0001\u0000\u0000\u0000\u01da\u01db\u0007\b\u0000\u0000\u01db"+ - "[\u0001\u0000\u0000\u0000\u01dc\u01dd\u0005\u0005\u0000\u0000\u01dd\u01de"+ - "\u0003^/\u0000\u01de]\u0001\u0000\u0000\u0000\u01df\u01e0\u0005?\u0000"+ - "\u0000\u01e0\u01e1\u0003\u0002\u0001\u0000\u01e1\u01e2\u0005@\u0000\u0000"+ - "\u01e2_\u0001\u0000\u0000\u0000\u01e3\u01e4\u0005\u000e\u0000\u0000\u01e4"+ - "\u01e8\u0005_\u0000\u0000\u01e5\u01e6\u0005\u000e\u0000\u0000\u01e6\u01e8"+ - "\u0005`\u0000\u0000\u01e7\u01e3\u0001\u0000\u0000\u0000\u01e7\u01e5\u0001"+ - "\u0000\u0000\u0000\u01e8a\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005\u0003"+ - "\u0000\u0000\u01ea\u01ed\u0005U\u0000\u0000\u01eb\u01ec\u0005S\u0000\u0000"+ - "\u01ec\u01ee\u0003.\u0017\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000\u01ed"+ - "\u01ee\u0001\u0000\u0000\u0000\u01ee\u01f8\u0001\u0000\u0000\u0000\u01ef"+ - "\u01f0\u0005T\u0000\u0000\u01f0\u01f5\u0003d2\u0000\u01f1\u01f2\u0005"+ - "!\u0000\u0000\u01f2\u01f4\u0003d2\u0000\u01f3\u01f1\u0001\u0000\u0000"+ - "\u0000\u01f4\u01f7\u0001\u0000\u0000\u0000\u01f5\u01f3\u0001\u0000\u0000"+ - "\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f9\u0001\u0000\u0000"+ - "\u0000\u01f7\u01f5\u0001\u0000\u0000\u0000\u01f8\u01ef\u0001\u0000\u0000"+ - "\u0000\u01f8\u01f9\u0001\u0000\u0000\u0000\u01f9c\u0001\u0000\u0000\u0000"+ - "\u01fa\u01fb\u0003.\u0017\u0000\u01fb\u01fc\u0005 \u0000\u0000\u01fc\u01fe"+ - "\u0001\u0000\u0000\u0000\u01fd\u01fa\u0001\u0000\u0000\u0000\u01fd\u01fe"+ - "\u0001\u0000\u0000\u0000\u01fe\u01ff\u0001\u0000\u0000\u0000\u01ff\u0200"+ - "\u0003.\u0017\u0000\u0200e\u0001\u0000\u0000\u00003qx\u0087\u0093\u009c"+ - "\u00a4\u00a8\u00b0\u00b2\u00b7\u00be\u00c3\u00ca\u00d0\u00d8\u00da\u00e4"+ - "\u00ee\u00f1\u00fd\u0105\u010d\u0111\u0115\u011d\u0129\u012d\u0133\u013c"+ - "\u0144\u014c\u015f\u016a\u0175\u017a\u0185\u018a\u018e\u0196\u019f\u01a8"+ - "\u01b3\u01c1\u01cc\u01cf\u01d4\u01e7\u01ed\u01f5\u01f8\u01fd"; + "\u0016\u0139\b\u0016\n\u0016\f\u0016\u013c\t\u0016\u0001\u0017\u0001\u0017"+ + "\u0001\u0017\u0005\u0017\u0141\b\u0017\n\u0017\f\u0017\u0144\t\u0017\u0001"+ + "\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001"+ + "\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001"+ + "\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u0157"+ + "\b\u001a\n\u001a\f\u001a\u015a\t\u001a\u0001\u001a\u0001\u001a\u0001\u001a"+ + "\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u0162\b\u001a\n\u001a"+ + "\f\u001a\u0165\t\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a"+ + "\u0001\u001a\u0001\u001a\u0005\u001a\u016d\b\u001a\n\u001a\f\u001a\u0170"+ + "\t\u001a\u0001\u001a\u0001\u001a\u0003\u001a\u0174\b\u001a\u0001\u001b"+ + "\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c"+ + "\u0005\u001c\u017d\b\u001c\n\u001c\f\u001c\u0180\t\u001c\u0001\u001d\u0001"+ + "\u001d\u0003\u001d\u0184\b\u001d\u0001\u001d\u0001\u001d\u0003\u001d\u0188"+ + "\b\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0005\u001e\u018e"+ + "\b\u001e\n\u001e\f\u001e\u0191\t\u001e\u0001\u001f\u0001\u001f\u0001\u001f"+ + "\u0001\u001f\u0005\u001f\u0197\b\u001f\n\u001f\f\u001f\u019a\t\u001f\u0001"+ + " \u0001 \u0001 \u0001 \u0005 \u01a0\b \n \f \u01a3\t \u0001!\u0001!\u0001"+ + "!\u0001!\u0001\"\u0001\"\u0001\"\u0001\"\u0003\"\u01ad\b\"\u0001#\u0001"+ + "#\u0001#\u0001#\u0001$\u0001$\u0001$\u0001%\u0001%\u0001%\u0005%\u01b9"+ + "\b%\n%\f%\u01bc\t%\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001("+ + "\u0001(\u0003(\u01c6\b(\u0001)\u0003)\u01c9\b)\u0001)\u0001)\u0001*\u0003"+ + "*\u01ce\b*\u0001*\u0001*\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001"+ + "-\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0001/\u0003/\u01e1"+ + "\b/\u00010\u00010\u00010\u00010\u00030\u01e7\b0\u00010\u00010\u00010\u0001"+ + "0\u00050\u01ed\b0\n0\f0\u01f0\t0\u00030\u01f2\b0\u00011\u00011\u00011"+ + "\u00031\u01f7\b1\u00011\u00011\u00011\u0000\u0003\u0002\n\u00102\u0000"+ + "\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c"+ + "\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`b\u0000\b\u0001\u0000:;\u0001"+ + "\u0000<>\u0002\u0000BBGG\u0001\u0000AB\u0002\u0000\u001f\u001f\"\"\u0001"+ + "\u0000%&\u0002\u0000$$22\u0001\u000039\u0215\u0000d\u0001\u0000\u0000"+ + "\u0000\u0002g\u0001\u0000\u0000\u0000\u0004v\u0001\u0000\u0000\u0000\u0006"+ + "\u0085\u0001\u0000\u0000\u0000\b\u0087\u0001\u0000\u0000\u0000\n\u00a6"+ + "\u0001\u0000\u0000\u0000\f\u00c1\u0001\u0000\u0000\u0000\u000e\u00c8\u0001"+ + "\u0000\u0000\u0000\u0010\u00ce\u0001\u0000\u0000\u0000\u0012\u00e2\u0001"+ + "\u0000\u0000\u0000\u0014\u00e4\u0001\u0000\u0000\u0000\u0016\u00f3\u0001"+ + "\u0000\u0000\u0000\u0018\u00f6\u0001\u0000\u0000\u0000\u001a\u0103\u0001"+ + "\u0000\u0000\u0000\u001c\u0105\u0001\u0000\u0000\u0000\u001e\u0113\u0001"+ + "\u0000\u0000\u0000 \u0115\u0001\u0000\u0000\u0000\"\u011e\u0001\u0000"+ + "\u0000\u0000$\u0122\u0001\u0000\u0000\u0000&\u0125\u0001\u0000\u0000\u0000"+ + "(\u012d\u0001\u0000\u0000\u0000*\u0133\u0001\u0000\u0000\u0000,\u0135"+ + "\u0001\u0000\u0000\u0000.\u013d\u0001\u0000\u0000\u00000\u0145\u0001\u0000"+ + "\u0000\u00002\u0147\u0001\u0000\u0000\u00004\u0173\u0001\u0000\u0000\u0000"+ + "6\u0175\u0001\u0000\u0000\u00008\u0178\u0001\u0000\u0000\u0000:\u0181"+ + "\u0001\u0000\u0000\u0000<\u0189\u0001\u0000\u0000\u0000>\u0192\u0001\u0000"+ + "\u0000\u0000@\u019b\u0001\u0000\u0000\u0000B\u01a4\u0001\u0000\u0000\u0000"+ + "D\u01a8\u0001\u0000\u0000\u0000F\u01ae\u0001\u0000\u0000\u0000H\u01b2"+ + "\u0001\u0000\u0000\u0000J\u01b5\u0001\u0000\u0000\u0000L\u01bd\u0001\u0000"+ + "\u0000\u0000N\u01c1\u0001\u0000\u0000\u0000P\u01c5\u0001\u0000\u0000\u0000"+ + "R\u01c8\u0001\u0000\u0000\u0000T\u01cd\u0001\u0000\u0000\u0000V\u01d1"+ + "\u0001\u0000\u0000\u0000X\u01d3\u0001\u0000\u0000\u0000Z\u01d5\u0001\u0000"+ + "\u0000\u0000\\\u01d8\u0001\u0000\u0000\u0000^\u01e0\u0001\u0000\u0000"+ + "\u0000`\u01e2\u0001\u0000\u0000\u0000b\u01f6\u0001\u0000\u0000\u0000d"+ + "e\u0003\u0002\u0001\u0000ef\u0005\u0000\u0000\u0001f\u0001\u0001\u0000"+ + "\u0000\u0000gh\u0006\u0001\uffff\uffff\u0000hi\u0003\u0004\u0002\u0000"+ + "io\u0001\u0000\u0000\u0000jk\n\u0001\u0000\u0000kl\u0005\u0019\u0000\u0000"+ + "ln\u0003\u0006\u0003\u0000mj\u0001\u0000\u0000\u0000nq\u0001\u0000\u0000"+ + "\u0000om\u0001\u0000\u0000\u0000op\u0001\u0000\u0000\u0000p\u0003\u0001"+ + "\u0000\u0000\u0000qo\u0001\u0000\u0000\u0000rw\u0003Z-\u0000sw\u0003\u001c"+ + "\u000e\u0000tw\u0003\u0016\u000b\u0000uw\u0003^/\u0000vr\u0001\u0000\u0000"+ + "\u0000vs\u0001\u0000\u0000\u0000vt\u0001\u0000\u0000\u0000vu\u0001\u0000"+ + "\u0000\u0000w\u0005\u0001\u0000\u0000\u0000x\u0086\u0003$\u0012\u0000"+ + "y\u0086\u0003(\u0014\u0000z\u0086\u00036\u001b\u0000{\u0086\u0003<\u001e"+ + "\u0000|\u0086\u00038\u001c\u0000}\u0086\u0003&\u0013\u0000~\u0086\u0003"+ + "\b\u0004\u0000\u007f\u0086\u0003>\u001f\u0000\u0080\u0086\u0003@ \u0000"+ + "\u0081\u0086\u0003D\"\u0000\u0082\u0086\u0003F#\u0000\u0083\u0086\u0003"+ + "`0\u0000\u0084\u0086\u0003H$\u0000\u0085x\u0001\u0000\u0000\u0000\u0085"+ + "y\u0001\u0000\u0000\u0000\u0085z\u0001\u0000\u0000\u0000\u0085{\u0001"+ + "\u0000\u0000\u0000\u0085|\u0001\u0000\u0000\u0000\u0085}\u0001\u0000\u0000"+ + "\u0000\u0085~\u0001\u0000\u0000\u0000\u0085\u007f\u0001\u0000\u0000\u0000"+ + "\u0085\u0080\u0001\u0000\u0000\u0000\u0085\u0081\u0001\u0000\u0000\u0000"+ + "\u0085\u0082\u0001\u0000\u0000\u0000\u0085\u0083\u0001\u0000\u0000\u0000"+ + "\u0085\u0084\u0001\u0000\u0000\u0000\u0086\u0007\u0001\u0000\u0000\u0000"+ + "\u0087\u0088\u0005\u0011\u0000\u0000\u0088\u0089\u0003\n\u0005\u0000\u0089"+ + "\t\u0001\u0000\u0000\u0000\u008a\u008b\u0006\u0005\uffff\uffff\u0000\u008b"+ + "\u008c\u0005+\u0000\u0000\u008c\u00a7\u0003\n\u0005\u0007\u008d\u00a7"+ + "\u0003\u000e\u0007\u0000\u008e\u00a7\u0003\f\u0006\u0000\u008f\u0091\u0003"+ + "\u000e\u0007\u0000\u0090\u0092\u0005+\u0000\u0000\u0091\u0090\u0001\u0000"+ + "\u0000\u0000\u0091\u0092\u0001\u0000\u0000\u0000\u0092\u0093\u0001\u0000"+ + "\u0000\u0000\u0093\u0094\u0005(\u0000\u0000\u0094\u0095\u0005\'\u0000"+ + "\u0000\u0095\u009a\u0003\u000e\u0007\u0000\u0096\u0097\u0005!\u0000\u0000"+ + "\u0097\u0099\u0003\u000e\u0007\u0000\u0098\u0096\u0001\u0000\u0000\u0000"+ + "\u0099\u009c\u0001\u0000\u0000\u0000\u009a\u0098\u0001\u0000\u0000\u0000"+ + "\u009a\u009b\u0001\u0000\u0000\u0000\u009b\u009d\u0001\u0000\u0000\u0000"+ + "\u009c\u009a\u0001\u0000\u0000\u0000\u009d\u009e\u00051\u0000\u0000\u009e"+ + "\u00a7\u0001\u0000\u0000\u0000\u009f\u00a0\u0003\u000e\u0007\u0000\u00a0"+ + "\u00a2\u0005)\u0000\u0000\u00a1\u00a3\u0005+\u0000\u0000\u00a2\u00a1\u0001"+ + "\u0000\u0000\u0000\u00a2\u00a3\u0001\u0000\u0000\u0000\u00a3\u00a4\u0001"+ + "\u0000\u0000\u0000\u00a4\u00a5\u0005,\u0000\u0000\u00a5\u00a7\u0001\u0000"+ + "\u0000\u0000\u00a6\u008a\u0001\u0000\u0000\u0000\u00a6\u008d\u0001\u0000"+ + "\u0000\u0000\u00a6\u008e\u0001\u0000\u0000\u0000\u00a6\u008f\u0001\u0000"+ + "\u0000\u0000\u00a6\u009f\u0001\u0000\u0000\u0000\u00a7\u00b0\u0001\u0000"+ + "\u0000\u0000\u00a8\u00a9\n\u0004\u0000\u0000\u00a9\u00aa\u0005\u001e\u0000"+ + "\u0000\u00aa\u00af\u0003\n\u0005\u0005\u00ab\u00ac\n\u0003\u0000\u0000"+ + "\u00ac\u00ad\u0005.\u0000\u0000\u00ad\u00af\u0003\n\u0005\u0004\u00ae"+ + "\u00a8\u0001\u0000\u0000\u0000\u00ae\u00ab\u0001\u0000\u0000\u0000\u00af"+ + "\u00b2\u0001\u0000\u0000\u0000\u00b0\u00ae\u0001\u0000\u0000\u0000\u00b0"+ + "\u00b1\u0001\u0000\u0000\u0000\u00b1\u000b\u0001\u0000\u0000\u0000\u00b2"+ + "\u00b0\u0001\u0000\u0000\u0000\u00b3\u00b5\u0003\u000e\u0007\u0000\u00b4"+ + "\u00b6\u0005+\u0000\u0000\u00b5\u00b4\u0001\u0000\u0000\u0000\u00b5\u00b6"+ + "\u0001\u0000\u0000\u0000\u00b6\u00b7\u0001\u0000\u0000\u0000\u00b7\u00b8"+ + "\u0005*\u0000\u0000\u00b8\u00b9\u0003V+\u0000\u00b9\u00c2\u0001\u0000"+ + "\u0000\u0000\u00ba\u00bc\u0003\u000e\u0007\u0000\u00bb\u00bd\u0005+\u0000"+ + "\u0000\u00bc\u00bb\u0001\u0000\u0000\u0000\u00bc\u00bd\u0001\u0000\u0000"+ + "\u0000\u00bd\u00be\u0001\u0000\u0000\u0000\u00be\u00bf\u00050\u0000\u0000"+ + "\u00bf\u00c0\u0003V+\u0000\u00c0\u00c2\u0001\u0000\u0000\u0000\u00c1\u00b3"+ + "\u0001\u0000\u0000\u0000\u00c1\u00ba\u0001\u0000\u0000\u0000\u00c2\r\u0001"+ + "\u0000\u0000\u0000\u00c3\u00c9\u0003\u0010\b\u0000\u00c4\u00c5\u0003\u0010"+ + "\b\u0000\u00c5\u00c6\u0003X,\u0000\u00c6\u00c7\u0003\u0010\b\u0000\u00c7"+ + "\u00c9\u0001\u0000\u0000\u0000\u00c8\u00c3\u0001\u0000\u0000\u0000\u00c8"+ + "\u00c4\u0001\u0000\u0000\u0000\u00c9\u000f\u0001\u0000\u0000\u0000\u00ca"+ + "\u00cb\u0006\b\uffff\uffff\u0000\u00cb\u00cf\u0003\u0012\t\u0000\u00cc"+ + "\u00cd\u0007\u0000\u0000\u0000\u00cd\u00cf\u0003\u0010\b\u0003\u00ce\u00ca"+ + "\u0001\u0000\u0000\u0000\u00ce\u00cc\u0001\u0000\u0000\u0000\u00cf\u00d8"+ + "\u0001\u0000\u0000\u0000\u00d0\u00d1\n\u0002\u0000\u0000\u00d1\u00d2\u0007"+ + "\u0001\u0000\u0000\u00d2\u00d7\u0003\u0010\b\u0003\u00d3\u00d4\n\u0001"+ + "\u0000\u0000\u00d4\u00d5\u0007\u0000\u0000\u0000\u00d5\u00d7\u0003\u0010"+ + "\b\u0002\u00d6\u00d0\u0001\u0000\u0000\u0000\u00d6\u00d3\u0001\u0000\u0000"+ + "\u0000\u00d7\u00da\u0001\u0000\u0000\u0000\u00d8\u00d6\u0001\u0000\u0000"+ + "\u0000\u00d8\u00d9\u0001\u0000\u0000\u0000\u00d9\u0011\u0001\u0000\u0000"+ + "\u0000\u00da\u00d8\u0001\u0000\u0000\u0000\u00db\u00e3\u00034\u001a\u0000"+ + "\u00dc\u00e3\u0003,\u0016\u0000\u00dd\u00e3\u0003\u0014\n\u0000\u00de"+ + "\u00df\u0005\'\u0000\u0000\u00df\u00e0\u0003\n\u0005\u0000\u00e0\u00e1"+ + "\u00051\u0000\u0000\u00e1\u00e3\u0001\u0000\u0000\u0000\u00e2\u00db\u0001"+ + "\u0000\u0000\u0000\u00e2\u00dc\u0001\u0000\u0000\u0000\u00e2\u00dd\u0001"+ + "\u0000\u0000\u0000\u00e2\u00de\u0001\u0000\u0000\u0000\u00e3\u0013\u0001"+ + "\u0000\u0000\u0000\u00e4\u00e5\u00030\u0018\u0000\u00e5\u00ef\u0005\'"+ + "\u0000\u0000\u00e6\u00f0\u0005<\u0000\u0000\u00e7\u00ec\u0003\n\u0005"+ + "\u0000\u00e8\u00e9\u0005!\u0000\u0000\u00e9\u00eb\u0003\n\u0005\u0000"+ + "\u00ea\u00e8\u0001\u0000\u0000\u0000\u00eb\u00ee\u0001\u0000\u0000\u0000"+ + "\u00ec\u00ea\u0001\u0000\u0000\u0000\u00ec\u00ed\u0001\u0000\u0000\u0000"+ + "\u00ed\u00f0\u0001\u0000\u0000\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000"+ + "\u00ef\u00e6\u0001\u0000\u0000\u0000\u00ef\u00e7\u0001\u0000\u0000\u0000"+ + "\u00ef\u00f0\u0001\u0000\u0000\u0000\u00f0\u00f1\u0001\u0000\u0000\u0000"+ + "\u00f1\u00f2\u00051\u0000\u0000\u00f2\u0015\u0001\u0000\u0000\u0000\u00f3"+ + "\u00f4\u0005\r\u0000\u0000\u00f4\u00f5\u0003\u0018\f\u0000\u00f5\u0017"+ + "\u0001\u0000\u0000\u0000\u00f6\u00fb\u0003\u001a\r\u0000\u00f7\u00f8\u0005"+ + "!\u0000\u0000\u00f8\u00fa\u0003\u001a\r\u0000\u00f9\u00f7\u0001\u0000"+ + "\u0000\u0000\u00fa\u00fd\u0001\u0000\u0000\u0000\u00fb\u00f9\u0001\u0000"+ + "\u0000\u0000\u00fb\u00fc\u0001\u0000\u0000\u0000\u00fc\u0019\u0001\u0000"+ + "\u0000\u0000\u00fd\u00fb\u0001\u0000\u0000\u0000\u00fe\u0104\u0003\n\u0005"+ + "\u0000\u00ff\u0100\u0003,\u0016\u0000\u0100\u0101\u0005 \u0000\u0000\u0101"+ + "\u0102\u0003\n\u0005\u0000\u0102\u0104\u0001\u0000\u0000\u0000\u0103\u00fe"+ + "\u0001\u0000\u0000\u0000\u0103\u00ff\u0001\u0000\u0000\u0000\u0104\u001b"+ + "\u0001\u0000\u0000\u0000\u0105\u0106\u0005\u0006\u0000\u0000\u0106\u010b"+ + "\u0003*\u0015\u0000\u0107\u0108\u0005!\u0000\u0000\u0108\u010a\u0003*"+ + "\u0015\u0000\u0109\u0107\u0001\u0000\u0000\u0000\u010a\u010d\u0001\u0000"+ + "\u0000\u0000\u010b\u0109\u0001\u0000\u0000\u0000\u010b\u010c\u0001\u0000"+ + "\u0000\u0000\u010c\u010f\u0001\u0000\u0000\u0000\u010d\u010b\u0001\u0000"+ + "\u0000\u0000\u010e\u0110\u0003\u001e\u000f\u0000\u010f\u010e\u0001\u0000"+ + "\u0000\u0000\u010f\u0110\u0001\u0000\u0000\u0000\u0110\u001d\u0001\u0000"+ + "\u0000\u0000\u0111\u0114\u0003 \u0010\u0000\u0112\u0114\u0003\"\u0011"+ + "\u0000\u0113\u0111\u0001\u0000\u0000\u0000\u0113\u0112\u0001\u0000\u0000"+ + "\u0000\u0114\u001f\u0001\u0000\u0000\u0000\u0115\u0116\u0005F\u0000\u0000"+ + "\u0116\u011b\u0003*\u0015\u0000\u0117\u0118\u0005!\u0000\u0000\u0118\u011a"+ + "\u0003*\u0015\u0000\u0119\u0117\u0001\u0000\u0000\u0000\u011a\u011d\u0001"+ + "\u0000\u0000\u0000\u011b\u0119\u0001\u0000\u0000\u0000\u011b\u011c\u0001"+ + "\u0000\u0000\u0000\u011c!\u0001\u0000\u0000\u0000\u011d\u011b\u0001\u0000"+ + "\u0000\u0000\u011e\u011f\u0005?\u0000\u0000\u011f\u0120\u0003 \u0010\u0000"+ + "\u0120\u0121\u0005@\u0000\u0000\u0121#\u0001\u0000\u0000\u0000\u0122\u0123"+ + "\u0005\u0004\u0000\u0000\u0123\u0124\u0003\u0018\f\u0000\u0124%\u0001"+ + "\u0000\u0000\u0000\u0125\u0127\u0005\u0010\u0000\u0000\u0126\u0128\u0003"+ + "\u0018\f\u0000\u0127\u0126\u0001\u0000\u0000\u0000\u0127\u0128\u0001\u0000"+ + "\u0000\u0000\u0128\u012b\u0001\u0000\u0000\u0000\u0129\u012a\u0005\u001d"+ + "\u0000\u0000\u012a\u012c\u0003\u0018\f\u0000\u012b\u0129\u0001\u0000\u0000"+ + "\u0000\u012b\u012c\u0001\u0000\u0000\u0000\u012c\'\u0001\u0000\u0000\u0000"+ + "\u012d\u012e\u0005\b\u0000\u0000\u012e\u0131\u0003\u0018\f\u0000\u012f"+ + "\u0130\u0005\u001d\u0000\u0000\u0130\u0132\u0003\u0018\f\u0000\u0131\u012f"+ + "\u0001\u0000\u0000\u0000\u0131\u0132\u0001\u0000\u0000\u0000\u0132)\u0001"+ + "\u0000\u0000\u0000\u0133\u0134\u0007\u0002\u0000\u0000\u0134+\u0001\u0000"+ + "\u0000\u0000\u0135\u013a\u00030\u0018\u0000\u0136\u0137\u0005#\u0000\u0000"+ + "\u0137\u0139\u00030\u0018\u0000\u0138\u0136\u0001\u0000\u0000\u0000\u0139"+ + "\u013c\u0001\u0000\u0000\u0000\u013a\u0138\u0001\u0000\u0000\u0000\u013a"+ + "\u013b\u0001\u0000\u0000\u0000\u013b-\u0001\u0000\u0000\u0000\u013c\u013a"+ + "\u0001\u0000\u0000\u0000\u013d\u0142\u00032\u0019\u0000\u013e\u013f\u0005"+ + "#\u0000\u0000\u013f\u0141\u00032\u0019\u0000\u0140\u013e\u0001\u0000\u0000"+ + "\u0000\u0141\u0144\u0001\u0000\u0000\u0000\u0142\u0140\u0001\u0000\u0000"+ + "\u0000\u0142\u0143\u0001\u0000\u0000\u0000\u0143/\u0001\u0000\u0000\u0000"+ + "\u0144\u0142\u0001\u0000\u0000\u0000\u0145\u0146\u0007\u0003\u0000\u0000"+ + "\u01461\u0001\u0000\u0000\u0000\u0147\u0148\u0005K\u0000\u0000\u01483"+ + "\u0001\u0000\u0000\u0000\u0149\u0174\u0005,\u0000\u0000\u014a\u014b\u0003"+ + "T*\u0000\u014b\u014c\u0005A\u0000\u0000\u014c\u0174\u0001\u0000\u0000"+ + "\u0000\u014d\u0174\u0003R)\u0000\u014e\u0174\u0003T*\u0000\u014f\u0174"+ + "\u0003N\'\u0000\u0150\u0174\u0005/\u0000\u0000\u0151\u0174\u0003V+\u0000"+ + "\u0152\u0153\u0005?\u0000\u0000\u0153\u0158\u0003P(\u0000\u0154\u0155"+ + "\u0005!\u0000\u0000\u0155\u0157\u0003P(\u0000\u0156\u0154\u0001\u0000"+ + "\u0000\u0000\u0157\u015a\u0001\u0000\u0000\u0000\u0158\u0156\u0001\u0000"+ + "\u0000\u0000\u0158\u0159\u0001\u0000\u0000\u0000\u0159\u015b\u0001\u0000"+ + "\u0000\u0000\u015a\u0158\u0001\u0000\u0000\u0000\u015b\u015c\u0005@\u0000"+ + "\u0000\u015c\u0174\u0001\u0000\u0000\u0000\u015d\u015e\u0005?\u0000\u0000"+ + "\u015e\u0163\u0003N\'\u0000\u015f\u0160\u0005!\u0000\u0000\u0160\u0162"+ + "\u0003N\'\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0165\u0001"+ + "\u0000\u0000\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001"+ + "\u0000\u0000\u0000\u0164\u0166\u0001\u0000\u0000\u0000\u0165\u0163\u0001"+ + "\u0000\u0000\u0000\u0166\u0167\u0005@\u0000\u0000\u0167\u0174\u0001\u0000"+ + "\u0000\u0000\u0168\u0169\u0005?\u0000\u0000\u0169\u016e\u0003V+\u0000"+ + "\u016a\u016b\u0005!\u0000\u0000\u016b\u016d\u0003V+\u0000\u016c\u016a"+ + "\u0001\u0000\u0000\u0000\u016d\u0170\u0001\u0000\u0000\u0000\u016e\u016c"+ + "\u0001\u0000\u0000\u0000\u016e\u016f\u0001\u0000\u0000\u0000\u016f\u0171"+ + "\u0001\u0000\u0000\u0000\u0170\u016e\u0001\u0000\u0000\u0000\u0171\u0172"+ + "\u0005@\u0000\u0000\u0172\u0174\u0001\u0000\u0000\u0000\u0173\u0149\u0001"+ + "\u0000\u0000\u0000\u0173\u014a\u0001\u0000\u0000\u0000\u0173\u014d\u0001"+ + "\u0000\u0000\u0000\u0173\u014e\u0001\u0000\u0000\u0000\u0173\u014f\u0001"+ + "\u0000\u0000\u0000\u0173\u0150\u0001\u0000\u0000\u0000\u0173\u0151\u0001"+ + "\u0000\u0000\u0000\u0173\u0152\u0001\u0000\u0000\u0000\u0173\u015d\u0001"+ + "\u0000\u0000\u0000\u0173\u0168\u0001\u0000\u0000\u0000\u01745\u0001\u0000"+ + "\u0000\u0000\u0175\u0176\u0005\n\u0000\u0000\u0176\u0177\u0005\u001b\u0000"+ + "\u0000\u01777\u0001\u0000\u0000\u0000\u0178\u0179\u0005\u000f\u0000\u0000"+ + "\u0179\u017e\u0003:\u001d\u0000\u017a\u017b\u0005!\u0000\u0000\u017b\u017d"+ + "\u0003:\u001d\u0000\u017c\u017a\u0001\u0000\u0000\u0000\u017d\u0180\u0001"+ + "\u0000\u0000\u0000\u017e\u017c\u0001\u0000\u0000\u0000\u017e\u017f\u0001"+ + "\u0000\u0000\u0000\u017f9\u0001\u0000\u0000\u0000\u0180\u017e\u0001\u0000"+ + "\u0000\u0000\u0181\u0183\u0003\n\u0005\u0000\u0182\u0184\u0007\u0004\u0000"+ + "\u0000\u0183\u0182\u0001\u0000\u0000\u0000\u0183\u0184\u0001\u0000\u0000"+ + "\u0000\u0184\u0187\u0001\u0000\u0000\u0000\u0185\u0186\u0005-\u0000\u0000"+ + "\u0186\u0188\u0007\u0005\u0000\u0000\u0187\u0185\u0001\u0000\u0000\u0000"+ + "\u0187\u0188\u0001\u0000\u0000\u0000\u0188;\u0001\u0000\u0000\u0000\u0189"+ + "\u018a\u0005\t\u0000\u0000\u018a\u018f\u0003.\u0017\u0000\u018b\u018c"+ + "\u0005!\u0000\u0000\u018c\u018e\u0003.\u0017\u0000\u018d\u018b\u0001\u0000"+ + "\u0000\u0000\u018e\u0191\u0001\u0000\u0000\u0000\u018f\u018d\u0001\u0000"+ + "\u0000\u0000\u018f\u0190\u0001\u0000\u0000\u0000\u0190=\u0001\u0000\u0000"+ + "\u0000\u0191\u018f\u0001\u0000\u0000\u0000\u0192\u0193\u0005\u0002\u0000"+ + "\u0000\u0193\u0198\u0003.\u0017\u0000\u0194\u0195\u0005!\u0000\u0000\u0195"+ + "\u0197\u0003.\u0017\u0000\u0196\u0194\u0001\u0000\u0000\u0000\u0197\u019a"+ + "\u0001\u0000\u0000\u0000\u0198\u0196\u0001\u0000\u0000\u0000\u0198\u0199"+ + "\u0001\u0000\u0000\u0000\u0199?\u0001\u0000\u0000\u0000\u019a\u0198\u0001"+ + "\u0000\u0000\u0000\u019b\u019c\u0005\f\u0000\u0000\u019c\u01a1\u0003B"+ + "!\u0000\u019d\u019e\u0005!\u0000\u0000\u019e\u01a0\u0003B!\u0000\u019f"+ + "\u019d\u0001\u0000\u0000\u0000\u01a0\u01a3\u0001\u0000\u0000\u0000\u01a1"+ + "\u019f\u0001\u0000\u0000\u0000\u01a1\u01a2\u0001\u0000\u0000\u0000\u01a2"+ + "A\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000\u0000\u01a4\u01a5"+ + "\u0003.\u0017\u0000\u01a5\u01a6\u0005O\u0000\u0000\u01a6\u01a7\u0003."+ + "\u0017\u0000\u01a7C\u0001\u0000\u0000\u0000\u01a8\u01a9\u0005\u0001\u0000"+ + "\u0000\u01a9\u01aa\u0003\u0012\t\u0000\u01aa\u01ac\u0003V+\u0000\u01ab"+ + "\u01ad\u0003J%\u0000\u01ac\u01ab\u0001\u0000\u0000\u0000\u01ac\u01ad\u0001"+ + "\u0000\u0000\u0000\u01adE\u0001\u0000\u0000\u0000\u01ae\u01af\u0005\u0007"+ + "\u0000\u0000\u01af\u01b0\u0003\u0012\t\u0000\u01b0\u01b1\u0003V+\u0000"+ + "\u01b1G\u0001\u0000\u0000\u0000\u01b2\u01b3\u0005\u000b\u0000\u0000\u01b3"+ + "\u01b4\u0003,\u0016\u0000\u01b4I\u0001\u0000\u0000\u0000\u01b5\u01ba\u0003"+ + "L&\u0000\u01b6\u01b7\u0005!\u0000\u0000\u01b7\u01b9\u0003L&\u0000\u01b8"+ + "\u01b6\u0001\u0000\u0000\u0000\u01b9\u01bc\u0001\u0000\u0000\u0000\u01ba"+ + "\u01b8\u0001\u0000\u0000\u0000\u01ba\u01bb\u0001\u0000\u0000\u0000\u01bb"+ + "K\u0001\u0000\u0000\u0000\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bd\u01be"+ + "\u00030\u0018\u0000\u01be\u01bf\u0005 \u0000\u0000\u01bf\u01c0\u00034"+ + "\u001a\u0000\u01c0M\u0001\u0000\u0000\u0000\u01c1\u01c2\u0007\u0006\u0000"+ + "\u0000\u01c2O\u0001\u0000\u0000\u0000\u01c3\u01c6\u0003R)\u0000\u01c4"+ + "\u01c6\u0003T*\u0000\u01c5\u01c3\u0001\u0000\u0000\u0000\u01c5\u01c4\u0001"+ + "\u0000\u0000\u0000\u01c6Q\u0001\u0000\u0000\u0000\u01c7\u01c9\u0007\u0000"+ + "\u0000\u0000\u01c8\u01c7\u0001\u0000\u0000\u0000\u01c8\u01c9\u0001\u0000"+ + "\u0000\u0000\u01c9\u01ca\u0001\u0000\u0000\u0000\u01ca\u01cb\u0005\u001c"+ + "\u0000\u0000\u01cbS\u0001\u0000\u0000\u0000\u01cc\u01ce\u0007\u0000\u0000"+ + "\u0000\u01cd\u01cc\u0001\u0000\u0000\u0000\u01cd\u01ce\u0001\u0000\u0000"+ + "\u0000\u01ce\u01cf\u0001\u0000\u0000\u0000\u01cf\u01d0\u0005\u001b\u0000"+ + "\u0000\u01d0U\u0001\u0000\u0000\u0000\u01d1\u01d2\u0005\u001a\u0000\u0000"+ + "\u01d2W\u0001\u0000\u0000\u0000\u01d3\u01d4\u0007\u0007\u0000\u0000\u01d4"+ + "Y\u0001\u0000\u0000\u0000\u01d5\u01d6\u0005\u0005\u0000\u0000\u01d6\u01d7"+ + "\u0003\\.\u0000\u01d7[\u0001\u0000\u0000\u0000\u01d8\u01d9\u0005?\u0000"+ + "\u0000\u01d9\u01da\u0003\u0002\u0001\u0000\u01da\u01db\u0005@\u0000\u0000"+ + "\u01db]\u0001\u0000\u0000\u0000\u01dc\u01dd\u0005\u000e\u0000\u0000\u01dd"+ + "\u01e1\u0005_\u0000\u0000\u01de\u01df\u0005\u000e\u0000\u0000\u01df\u01e1"+ + "\u0005`\u0000\u0000\u01e0\u01dc\u0001\u0000\u0000\u0000\u01e0\u01de\u0001"+ + "\u0000\u0000\u0000\u01e1_\u0001\u0000\u0000\u0000\u01e2\u01e3\u0005\u0003"+ + "\u0000\u0000\u01e3\u01e6\u0005U\u0000\u0000\u01e4\u01e5\u0005S\u0000\u0000"+ + "\u01e5\u01e7\u0003.\u0017\u0000\u01e6\u01e4\u0001\u0000\u0000\u0000\u01e6"+ + "\u01e7\u0001\u0000\u0000\u0000\u01e7\u01f1\u0001\u0000\u0000\u0000\u01e8"+ + "\u01e9\u0005T\u0000\u0000\u01e9\u01ee\u0003b1\u0000\u01ea\u01eb\u0005"+ + "!\u0000\u0000\u01eb\u01ed\u0003b1\u0000\u01ec\u01ea\u0001\u0000\u0000"+ + "\u0000\u01ed\u01f0\u0001\u0000\u0000\u0000\u01ee\u01ec\u0001\u0000\u0000"+ + "\u0000\u01ee\u01ef\u0001\u0000\u0000\u0000\u01ef\u01f2\u0001\u0000\u0000"+ + "\u0000\u01f0\u01ee\u0001\u0000\u0000\u0000\u01f1\u01e8\u0001\u0000\u0000"+ + "\u0000\u01f1\u01f2\u0001\u0000\u0000\u0000\u01f2a\u0001\u0000\u0000\u0000"+ + "\u01f3\u01f4\u0003.\u0017\u0000\u01f4\u01f5\u0005 \u0000\u0000\u01f5\u01f7"+ + "\u0001\u0000\u0000\u0000\u01f6\u01f3\u0001\u0000\u0000\u0000\u01f6\u01f7"+ + "\u0001\u0000\u0000\u0000\u01f7\u01f8\u0001\u0000\u0000\u0000\u01f8\u01f9"+ + "\u0003.\u0017\u0000\u01f9c\u0001\u0000\u0000\u00002ov\u0085\u0091\u009a"+ + "\u00a2\u00a6\u00ae\u00b0\u00b5\u00bc\u00c1\u00c8\u00ce\u00d6\u00d8\u00e2"+ + "\u00ec\u00ef\u00fb\u0103\u010b\u010f\u0113\u011b\u0127\u012b\u0131\u013a"+ + "\u0142\u0158\u0163\u016e\u0173\u017e\u0183\u0187\u018f\u0198\u01a1\u01ac"+ + "\u01ba\u01c5\u01c8\u01cd\u01e0\u01e6\u01ee\u01f1\u01f6"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 3868929936a72..ffbcfc57a90ea 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -468,18 +468,6 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

    The default implementation does nothing.

    */ @Override public void exitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { } - /** - * {@inheritDoc} - * - *

    The default implementation does nothing.

    - */ - @Override public void enterIdPattern(EsqlBaseParser.IdPatternContext ctx) { } - /** - * {@inheritDoc} - * - *

    The default implementation does nothing.

    - */ - @Override public void exitIdPattern(EsqlBaseParser.IdPatternContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index 457f08c20fc81..ca876efb3e7da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -278,13 +278,6 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

    */ @Override public T visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

    The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

    - */ - @Override public T visitIdPattern(EsqlBaseParser.IdPatternContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index 5c238b26fd8a0..75656c1df76dc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -421,16 +421,6 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); - /** - * Enter a parse tree produced by {@link EsqlBaseParser#idPattern}. - * @param ctx the parse tree - */ - void enterIdPattern(EsqlBaseParser.IdPatternContext ctx); - /** - * Exit a parse tree produced by {@link EsqlBaseParser#idPattern}. - * @param ctx the parse tree - */ - void exitIdPattern(EsqlBaseParser.IdPatternContext ctx); /** * Enter a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link EsqlBaseParser#constant}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index 8bcdcfcee23bd..0c3cc791f7fe2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -255,12 +255,6 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); - /** - * Visit a parse tree produced by {@link EsqlBaseParser#idPattern}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitIdPattern(EsqlBaseParser.IdPatternContext ctx); /** * Visit a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link EsqlBaseParser#constant}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 4294fada8ad73..d58e25391aa26 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -233,11 +233,8 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat // check special wildcard case if (patterns.size() == 1) { var idCtx = patterns.get(0); - if (idCtx.idPattern().size() == 1) { - var idPattern = idCtx.idPattern(0); - if (idPattern.UNQUOTED_ID_PATTERN() != null && idPattern.UNQUOTED_ID_PATTERN().getText().equals(WILDCARD)) { - return new UnresolvedStar(src, null); - } + if (idCtx.ID_PATTERN().getText().equals(WILDCARD)) { + return new UnresolvedStar(src, null); } } @@ -245,25 +242,29 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat // Builds a list of either strings (which map verbatim) or Automatons which match any string List objects = new ArrayList<>(patterns.size()); for (int i = 0, s = patterns.size(); i < s; i++) { - var patternContext = patterns.get(i); - String name; if (i > 0) { patternString.append("."); nameString.append("."); objects.add("."); } - for (var idContext : patternContext.idPattern()) { - // a patternCtx can be a series of quoted and unquoted sections + + String patternContext = patterns.get(i).ID_PATTERN().getText(); + // as this is one big string of fragments mashed together, break it manually into quoted vs unquoted + // for readability reasons, do a first pass to break the string into fragments and then process each of them + // to avoid doing a string allocation + List fragments = breakIntoFragments(patternContext); + + for (var fragment : fragments) { + // unquoted fragment // a wildcard that matches can only appear in an unquoted section - if (idContext.UNQUOTED_ID_PATTERN() != null) { - name = idContext.UNQUOTED_ID_PATTERN().getText(); - patternString.append(name); - nameString.append(name); + if (fragment.charAt(0) == '`' == false) { + patternString.append(fragment); + nameString.append(fragment); // loop the string itself to extract any * and make them an automaton directly // the code is somewhat messy but doesn't invoke the full blown Regex engine either - if (Regex.isSimpleMatchPattern(name)) { + if (Regex.isSimpleMatchPattern(fragment)) { hasPattern = true; - String str = name; + String str = fragment; boolean keepGoing = false; do { int localIndex = str.indexOf('*'); @@ -292,14 +293,13 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat } } while (keepGoing); } else { - objects.add(name); + objects.add(fragment); } } // quoted - definitely no pattern else { - var quoted = idContext.QUOTED_IDENTIFIER(); - patternString.append(quoted.getText()); - var unquotedString = unquoteIdentifier(quoted, null); + patternString.append(fragment); + var unquotedString = unquoteIdString(fragment); objects.add(unquotedString); nameString.append(unquotedString); } @@ -327,6 +327,60 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat return result; } + static List breakIntoFragments(String idPattern) { + List fragments = new ArrayList<>(); + char backtick = '`'; + boolean inQuotes = false; + boolean keepGoing = true; + int from = 0, offset = -1; + + do { + offset = idPattern.indexOf(backtick, offset); + String fragment = null; + // unquoted fragment + if (offset < 0) { + keepGoing = false; + // pick trailing string + fragment = idPattern.substring(from); + } + // quoted fragment + else { + // if not in quotes + // copy the string over + // otherwise keep on going + if (inQuotes == false) { + inQuotes = true; + if (offset != 0) { + fragment = idPattern.substring(from, offset); + from = offset; + } + } // in quotes + else { + // if double backtick keep on going + var ahead = offset + 1; + if (ahead < idPattern.length() && idPattern.charAt(ahead) == backtick) { + // move offset + offset = ahead; + } + // otherwise end the quote + else { + inQuotes = false; + // include the quote + offset++; + fragment = idPattern.substring(from, offset); + from = offset; + } + } + // keep moving the offset + offset++; + } + if (fragment != null) { + fragments.add(fragment); + } + } while (keepGoing && offset <= idPattern.length()); + return fragments; + } + @Override public Object visitQualifiedIntegerLiteral(EsqlBaseParser.QualifiedIntegerLiteralContext ctx) { Source source = source(ctx); @@ -503,8 +557,8 @@ public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseCont private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternContext ctx) { var name = visitQualifiedNamePattern(ctx); - if (name != null && name.name().contains(WILDCARD)) { - throw new ParsingException(source(ctx), "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", name.name()); + if (name instanceof UnresolvedNamePattern up) { + throw new ParsingException(source(ctx), "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", up.pattern()); } return name; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java index 79bf2b1b2d4b5..67f8eb407ee11 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java @@ -31,14 +31,17 @@ public String visitFromIdentifier(FromIdentifierContext ctx) { protected static String unquoteIdentifier(TerminalNode quotedNode, TerminalNode unquotedNode) { String result; if (quotedNode != null) { - String identifier = quotedNode.getText(); - result = identifier.substring(1, identifier.length() - 1).replace("``", "`"); + result = unquoteIdString(quotedNode.getText()); } else { result = unquotedNode.getText(); } return result; } + protected static String unquoteIdString(String quotedString) { + return quotedString.substring(1, quotedString.length() - 1).replace("``", "`"); + } + public String visitFromIdentifiers(List ctx) { return Strings.collectionToDelimitedString(visitList(this, ctx, String.class), ","); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index e8366f571da8f..b1f8a4ed2f07d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -58,6 +58,7 @@ import java.util.function.Function; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; +import static org.elasticsearch.xpack.esql.parser.ExpressionBuilder.breakIntoFragments; import static org.elasticsearch.xpack.ql.expression.Literal.FALSE; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; import static org.elasticsearch.xpack.ql.expression.function.FunctionResolutionStrategy.DEFAULT; @@ -68,6 +69,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -874,6 +876,53 @@ private void assertIdentifierAsIndexPattern(String identifier, String statement) assertThat(table.table().index(), is(identifier)); } + public void testIdPatternUnquoted() throws Exception { + var string = "regularString"; + assertThat(breakIntoFragments(string), contains(string)); + } + + public void testIdPatternQuoted() throws Exception { + var string = "`escaped string`"; + assertThat(breakIntoFragments(string), contains(string)); + } + + public void testIdPatternQuotedWithDoubleBackticks() throws Exception { + var string = "`escaped``string`"; + assertThat(breakIntoFragments(string), contains(string)); + } + + public void testIdPatternUnquotedAndQuoted() throws Exception { + var string = "this`is`a`mix`of`ids`"; + assertThat(breakIntoFragments(string), contains("this", "`is`", "a", "`mix`", "of", "`ids`")); + } + + public void testIdPatternQuotedTraling() throws Exception { + var string = "`foo`*"; + assertThat(breakIntoFragments(string), contains("`foo`", "*")); + } + + public void testIdPatternWithDoubleQuotedStrings() throws Exception { + var string = "`this``is`a`quoted `` string``with`backticks"; + assertThat(breakIntoFragments(string), contains("`this``is`", "a", "`quoted `` string``with`", "backticks")); + } + + public void testSpaceNotAllowedInIdPattern() throws Exception { + expectError("ROW a = 1| RENAME a AS this is `not okay`", "mismatched input 'is' expecting {'.', 'as'}"); + } + + public void testSpaceNotAllowedInIdPatternKeep() throws Exception { + expectError("ROW a = 1, b = 1| KEEP a b", "extraneous input 'b'"); + } + + public void testEnrichOnMatchField() { + var plan = statement("ROW a = \"1\" | ENRICH languages_policy ON a WITH ```name``* = language_name`"); + var enrich = as(plan, Enrich.class); + var lists = enrich.enrichFields(); + assertThat(lists, hasSize(1)); + var ua = as(lists.get(0), UnresolvedAttribute.class); + assertThat(ua.name(), is("`name`* = language_name")); + } + private LogicalPlan statement(String e) { return statement(e, List.of()); } From 4857118a9e4f6dc19ef6f631d3f4fbf7d1069777 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 14 Feb 2024 15:24:49 -0800 Subject: [PATCH 44/78] Run heap attack tests with two nodes (#105110) We should run heap-attack tests on multiple nodes to ensure that we avoid causing OOM during the serialization/deserialization of exchange responses. I've merged the required changes and run thousands of iterations of these tests without seeing any failures. --- .../xpack/esql/heap_attack/HeapAttackIT.java | 46 +++++++++++++------ .../esql/heap_attack/HeapAttackPlugin.java | 20 ++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index 4e6e149b454e8..8c87ef5977114 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; @@ -59,15 +60,25 @@ public class HeapAttackIT extends ESRestTestCase { @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("test-esql-heap-attack") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .build(); + public static ElasticsearchCluster cluster = buildCluster(); static volatile boolean SUITE_ABORTED = false; + static ElasticsearchCluster buildCluster() { + var spec = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .nodes(2) + .module("test-esql-heap-attack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial"); + String javaVersion = JvmInfo.jvmInfo().version(); + if (javaVersion.equals("20") || javaVersion.equals("21")) { + // see https://github.com/elastic/elasticsearch/issues/99592 + spec.jvmArg("-XX:+UnlockDiagnosticVMOptions -XX:+G1UsePreventiveGC"); + } + return spec.build(); + } + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); @@ -526,27 +537,34 @@ private void initMvLongsIndex(int docs, int fields, int fieldValues) throws IOEx private void bulk(String name, String bulk) throws IOException { Request request = new Request("POST", "/" + name + "/_bulk"); request.addParameter("filter_path", "errors"); - request.setJsonEntity(bulk.toString()); + request.setJsonEntity(bulk); + request.setOptions( + RequestOptions.DEFAULT.toBuilder() + .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(5).millis())).build()) + ); Response response = client().performRequest(request); assertThat(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8), equalTo("{\"errors\":false}")); } private void initIndex(String name, String bulk) throws IOException { + if (indexExists(name) == false) { + // not strictly required, but this can help isolate failure from bulk indexing. + createIndex(name); + } if (hasText(bulk)) { bulk(name, bulk); } - - Request request = new Request("POST", "/" + name + "/_refresh"); - Response response = client().performRequest(request); - assertWriteResponse(response); - - request = new Request("POST", "/" + name + "/_forcemerge"); + Request request = new Request("POST", "/" + name + "/_forcemerge"); request.addParameter("max_num_segments", "1"); - response = client().performRequest(request); + RequestOptions.Builder requestOptions = RequestOptions.DEFAULT.toBuilder() + .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(5).millis())).build()); + request.setOptions(requestOptions); + Response response = client().performRequest(request); assertWriteResponse(response); request = new Request("POST", "/" + name + "/_refresh"); response = client().performRequest(request); + request.setOptions(requestOptions); assertWriteResponse(response); } diff --git a/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java b/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java index 224a3eccfef46..3ec11a614145c 100644 --- a/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java +++ b/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java @@ -20,8 +20,10 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -47,4 +49,22 @@ public List getRestHandlers( ) { return List.of(new RestTriggerOutOfMemoryAction()); } + + // Deliberately unregistered, only used in unit tests. Copied to AbstractSimpleTransportTestCase#IGNORE_DESERIALIZATION_ERRORS_SETTING + // so that tests in other packages can see it too. + static final Setting IGNORE_DESERIALIZATION_ERRORS_SETTING = Setting.boolSetting( + "transport.ignore_deserialization_errors", + true, + Setting.Property.NodeScope + ); + + @Override + public List> getSettings() { + return CollectionUtils.appendToCopy(super.getSettings(), IGNORE_DESERIALIZATION_ERRORS_SETTING); + } + + @Override + public Settings additionalSettings() { + return Settings.builder().put(super.additionalSettings()).put(IGNORE_DESERIALIZATION_ERRORS_SETTING.getKey(), true).build(); + } } From 2cec43d0a8e49ca3debde64084803799cab78313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 15 Feb 2024 08:35:14 +0100 Subject: [PATCH 45/78] [Transform] Fix a bug where destination index aliases are not set up for an unattended transform (#105499) --- docs/changelog/105499.yaml | 5 + .../integration/TransformChainIT.java | 33 +++++- .../TransformInsufficientPermissionsIT.java | 11 +- .../integration/TransformDestIndexIT.java | 106 +++++++++++++----- .../xpack/transform/Transform.java | 2 +- .../transforms/ClientTransformIndexer.java | 28 +++++ .../ClientTransformIndexerBuilder.java | 24 ++++ .../transforms/TransformIndexer.java | 15 ++- .../TransformPersistentTasksExecutor.java | 12 +- .../ClientTransformIndexerTests.java | 21 ++++ .../TransformIndexerFailureHandlingTests.java | 12 ++ ...IndexerFailureOnStatePersistenceTests.java | 18 +++ .../TransformIndexerStateTests.java | 10 ++ .../transforms/TransformIndexerTests.java | 5 + ...TransformPersistentTasksExecutorTests.java | 3 +- .../transforms/TransformTaskTests.java | 5 + 16 files changed, 266 insertions(+), 44 deletions(-) create mode 100644 docs/changelog/105499.yaml diff --git a/docs/changelog/105499.yaml b/docs/changelog/105499.yaml new file mode 100644 index 0000000000000..bfc297411efa7 --- /dev/null +++ b/docs/changelog/105499.yaml @@ -0,0 +1,5 @@ +pr: 105499 +summary: Fix a bug where destination index aliases are not set up for an unattended transform +area: Transform +type: bug +issues: [] diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java index 74ee8ea88e04f..600ceb3cd8202 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java @@ -38,7 +38,16 @@ public class TransformChainIT extends TransformRestTestCase { }, "dest": { "index": "%s", - "pipeline": "%s" + "pipeline": "%s", + "aliases": [ + { + "alias": "%s" + }, + { + "alias": "%s", + "move_on_creation": true + } + ] }, "sync": { "time": { @@ -172,9 +181,14 @@ private void testChainedTransforms(final int numTransforms) throws Exception { // The number of documents is expected to be the same in all the indices. String sourceIndex = i == 0 ? reviewsIndexName : transformIds.get(i - 1) + "-dest"; String destIndex = transformId + "-dest"; + String destReadAlias = destIndex + "-read"; + String destWriteAlias = destIndex + "-write"; assertFalse(indexExists(destIndex)); + assertFalse(aliasExists(destReadAlias)); + assertFalse(aliasExists(destWriteAlias)); - assertAcknowledged(putTransform(transformId, createTransformConfig(sourceIndex, destIndex), true, RequestOptions.DEFAULT)); + String transformConfig = createTransformConfig(sourceIndex, destIndex, destReadAlias, destWriteAlias); + assertAcknowledged(putTransform(transformId, transformConfig, true, RequestOptions.DEFAULT)); } List transformIdsShuffled = new ArrayList<>(transformIds); @@ -198,6 +212,17 @@ private void testChainedTransforms(final int numTransforms) throws Exception { } }, 60, TimeUnit.SECONDS); + for (String transformId : transformIds) { + String destIndex = transformId + "-dest"; + String destReadAlias = destIndex + "-read"; + String destWriteAlias = destIndex + "-write"; + // Verify that the destination index is created. + assertTrue(indexExists(destIndex)); + // Verify that the destination index aliases are set up. + assertTrue(aliasExists(destReadAlias)); + assertTrue(aliasExists(destWriteAlias)); + } + // Stop all the transforms. for (String transformId : transformIds) { stopTransform(transformId); @@ -208,12 +233,14 @@ private void testChainedTransforms(final int numTransforms) throws Exception { } } - private static String createTransformConfig(String sourceIndex, String destIndex) { + private static String createTransformConfig(String sourceIndex, String destIndex, String destReadAlias, String destWriteAlias) { return Strings.format( TRANSFORM_CONFIG_TEMPLATE, sourceIndex, destIndex, SET_INGEST_TIME_PIPELINE, + destReadAlias, + destWriteAlias, "1s", "1s", randomBoolean(), diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java index e46d32295078f..dc48ceb7b309b 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java @@ -420,8 +420,14 @@ public void testTransformPermissionsDeferUnattendedNoDest() throws Exception { startTransform(config.getId(), RequestOptions.DEFAULT); - // transform is red due to lacking permissions - assertRed(transformId, authIssue); + // Give the transform indexer enough time to try creating destination index + Thread.sleep(5_000); + + String destIndexIssue = Strings.format("Could not create destination index [%s] for transform [%s]", destIndexName, transformId); + // transform's auth state status is still RED due to: + // - lacking permissions + // - and the inability to create destination index in the indexer (which is also a consequence of lacking permissions) + assertRed(transformId, authIssue, destIndexIssue); // update transform's credentials so that the transform has permission to access source/dest indices updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build()); @@ -576,6 +582,7 @@ private void assertGreen(String transformId) throws IOException { assertThat("Stats were: " + stats, extractValue(stats, "health", "issues"), is(nullValue())); } + // We expect exactly the issues passed as "expectedHealthIssueDetails". Not more, not less. @SuppressWarnings("unchecked") private void assertRed(String transformId, String... expectedHealthIssueDetails) throws IOException { Map stats = getBasicTransformStats(transformId); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java index 29576231d848c..4527fefde4b02 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDestIndexIT.java @@ -24,6 +24,7 @@ import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; public class TransformDestIndexIT extends TransformRestTestCase { @@ -114,32 +115,26 @@ public void testTransformDestIndexAliases() throws Exception { assertAliases(destIndex2, destAliasAll, destAliasLatest); } - public void testTransformDestIndexCreatedDuringUpdate_NoDeferValidation() throws Exception { - testTransformDestIndexCreatedDuringUpdate(false); + public void testUnattendedTransformDestIndexCreatedDuringUpdate_NoDeferValidation() throws Exception { + testUnattendedTransformDestIndexCreatedDuringUpdate(false); } - public void testTransformDestIndexCreatedDuringUpdate_DeferValidation() throws Exception { - testTransformDestIndexCreatedDuringUpdate(true); + public void testUnattendedTransformDestIndexCreatedDuringUpdate_DeferValidation() throws Exception { + testUnattendedTransformDestIndexCreatedDuringUpdate(true); } - private void testTransformDestIndexCreatedDuringUpdate(boolean deferValidation) throws Exception { + private void testUnattendedTransformDestIndexCreatedDuringUpdate(boolean deferValidation) throws Exception { String transformId = "test_dest_index_on_update" + (deferValidation ? "-defer" : ""); String destIndex = transformId + "-dest"; + String destAliasAll = transformId + ".all"; + String destAliasLatest = transformId + ".latest"; + List destAliases = List.of(new DestAlias(destAliasAll, false), new DestAlias(destAliasLatest, true)); + // Create and start the unattended transform + SettingsConfig settingsConfig = new SettingsConfig.Builder().setUnattended(true).build(); + createPivotReviewsTransform(transformId, destIndex, null, null, destAliases, settingsConfig, null, null, REVIEWS_INDEX_NAME); assertFalse(indexExists(destIndex)); - // Create and start the unattended transform - createPivotReviewsTransform( - transformId, - destIndex, - null, - null, - null, - new SettingsConfig.Builder().setUnattended(true).build(), - null, - null, - REVIEWS_INDEX_NAME - ); startTransform(transformId); // Update the unattended transform. This will trigger destination index creation. @@ -151,6 +146,66 @@ private void testTransformDestIndexCreatedDuringUpdate(boolean deferValidation) // Verify that the destination index now exists assertTrue(indexExists(destIndex)); + // Verify that both aliases are configured on the dest index + assertAliases(destIndex, destAliasAll, destAliasLatest); + } + + public void testUnattendedTransformDestIndexCreatedDuringUpdate_EmptySourceIndex_NoDeferValidation() throws Exception { + testUnattendedTransformDestIndexCreatedDuringUpdate_EmptySourceIndex(false); + } + + public void testUnattendedTransformDestIndexCreatedDuringUpdate_EmptySourceIndex_DeferValidation() throws Exception { + testUnattendedTransformDestIndexCreatedDuringUpdate_EmptySourceIndex(true); + } + + private void testUnattendedTransformDestIndexCreatedDuringUpdate_EmptySourceIndex(boolean deferValidation) throws Exception { + String transformId = "test_dest_index_on_update-empty" + (deferValidation ? "-defer" : ""); + String sourceIndexIndex = transformId + "-src"; + String destIndex = transformId + "-dest"; + String destAliasAll = transformId + ".all"; + String destAliasLatest = transformId + ".latest"; + List destAliases = List.of(new DestAlias(destAliasAll, false), new DestAlias(destAliasLatest, true)); + + // We want to use an empty source index to make sure transform will not write to the destination index + putReviewsIndex(sourceIndexIndex, "date", false); + assertFalse(indexExists(destIndex)); + + // Create and start the unattended transform + SettingsConfig settingsConfig = new SettingsConfig.Builder().setUnattended(true).build(); + createPivotReviewsTransform(transformId, destIndex, null, null, destAliases, settingsConfig, null, null, sourceIndexIndex); + startTransform(transformId); + + // Verify that the destination index creation got skipped + assertFalse(indexExists(destIndex)); + + // Update the unattended transform. This will trigger destination index creation. + // The update has to change something in the config (here, max_page_search_size). Otherwise it would have been optimized away. + updateTransform(transformId, """ + { "settings": { "max_page_search_size": 123 } }""", deferValidation); + + // Verify that the destination index now exists + assertTrue(indexExists(destIndex)); + // Verify that both aliases are configured on the dest index + assertAliases(destIndex, destAliasAll, destAliasLatest); + } + + public void testUnattendedTransformDestIndexCreatedByIndexer() throws Exception { + String transformId = "test_dest_index_in_indexer"; + String destIndex = transformId + "-dest"; + String destAliasAll = transformId + ".all"; + String destAliasLatest = transformId + ".latest"; + List destAliases = List.of(new DestAlias(destAliasAll, false), new DestAlias(destAliasLatest, true)); + + // Create and start the unattended transform + SettingsConfig settingsConfig = new SettingsConfig.Builder().setUnattended(true).build(); + createPivotReviewsTransform(transformId, destIndex, null, null, destAliases, settingsConfig, null, null, REVIEWS_INDEX_NAME); + startTransform(transformId); + waitForTransformCheckpoint(transformId, 1); + + // Verify that the destination index exists + assertTrue(indexExists(destIndex)); + // Verify that both aliases are configured on the dest index + assertAliases(destIndex, destAliasAll, destAliasLatest); } public void testTransformDestIndexMappings_DeduceMappings() throws Exception { @@ -245,20 +300,9 @@ private void testTransformDestIndexMappings(String transformId, boolean deduceMa assertTrue(indexExists(destIndex)); assertThat( getIndexMappingAsMap(destIndex), - is( - equalTo( - Map.of( - "properties", - Map.of( - "avg_rating", - Map.of("type", "double"), - "reviewer", - Map.of("type", "keyword"), - "timestamp", - Map.of("type", "date") - ) - ) - ) + hasEntry( + "properties", + Map.of("avg_rating", Map.of("type", "double"), "reviewer", Map.of("type", "keyword"), "timestamp", Map.of("type", "date")) ) ); Map searchResult = getAsMap(destIndex + "/_search?q=reviewer:user_0"); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index bde5576fb0c92..54a7c9ec733c2 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -283,7 +283,7 @@ public List> getPersistentTasksExecutor( threadPool, clusterService, settingsModule.getSettings(), - getTransformExtension().getTransformInternalIndexAdditionalSettings(), + getTransformExtension(), expressionResolver ) ); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index 1f9ec86ada8e2..1634f417924c0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -27,7 +27,10 @@ import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -54,9 +57,11 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; +import org.elasticsearch.xpack.transform.TransformExtension; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex; +import org.elasticsearch.xpack.transform.persistence.TransformIndex; import org.elasticsearch.xpack.transform.transforms.pivot.SchemaUtil; import org.elasticsearch.xpack.transform.utils.ExceptionRootCauseFinder; @@ -75,6 +80,9 @@ class ClientTransformIndexer extends TransformIndexer { private static final Logger logger = LogManager.getLogger(ClientTransformIndexer.class); private final ParentTaskAssigningClient client; + private final ClusterService clusterService; + private final IndexNameExpressionResolver indexNameExpressionResolver; + private final Settings destIndexSettings; private final AtomicBoolean oldStatsCleanedUp = new AtomicBoolean(false); private final AtomicReference seqNoPrimaryTermAndIndexHolder; @@ -84,6 +92,9 @@ class ClientTransformIndexer extends TransformIndexer { ClientTransformIndexer( ThreadPool threadPool, + ClusterService clusterService, + IndexNameExpressionResolver indexNameExpressionResolver, + TransformExtension transformExtension, TransformServices transformServices, CheckpointProvider checkpointProvider, AtomicReference initialState, @@ -112,6 +123,9 @@ class ClientTransformIndexer extends TransformIndexer { context ); this.client = ExceptionsHelper.requireNonNull(client, "client"); + this.clusterService = clusterService; + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.destIndexSettings = transformExtension.getTransformDestinationIndexSettings(); this.seqNoPrimaryTermAndIndexHolder = new AtomicReference<>(seqNoPrimaryTermAndIndex); // TODO: move into context constructor @@ -288,6 +302,20 @@ void doGetFieldMappings(ActionListener> fieldMappingsListene SchemaUtil.getDestinationFieldMappings(client, getConfig().getDestination().getIndex(), fieldMappingsListener); } + @Override + void doMaybeCreateDestIndex(Map deducedDestIndexMappings, ActionListener listener) { + TransformIndex.createDestinationIndex( + client, + auditor, + indexNameExpressionResolver, + clusterService.state(), + transformConfig, + destIndexSettings, + deducedDestIndexMappings, + listener + ); + } + void validate(ActionListener listener) { ClientHelper.executeAsyncWithOrigin( client, diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java index 402f2489698e2..815fc66694ea2 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.transform.transforms; import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; @@ -15,6 +17,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; +import org.elasticsearch.xpack.transform.TransformExtension; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex; @@ -23,6 +26,9 @@ class ClientTransformIndexerBuilder { private ParentTaskAssigningClient parentTaskClient; + private ClusterService clusterService; + private IndexNameExpressionResolver indexNameExpressionResolver; + private TransformExtension transformExtension; private TransformServices transformServices; private TransformConfig transformConfig; private TransformIndexerStats initialStats; @@ -44,6 +50,9 @@ ClientTransformIndexer build(ThreadPool threadPool, TransformContext context) { return new ClientTransformIndexer( threadPool, + clusterService, + indexNameExpressionResolver, + transformExtension, transformServices, checkpointProvider, new AtomicReference<>(this.indexerState), @@ -73,6 +82,21 @@ ClientTransformIndexerBuilder setClient(ParentTaskAssigningClient parentTaskClie return this; } + ClientTransformIndexerBuilder setIndexNameExpressionResolver(IndexNameExpressionResolver indexNameExpressionResolver) { + this.indexNameExpressionResolver = indexNameExpressionResolver; + return this; + } + + ClientTransformIndexerBuilder setClusterService(ClusterService clusterService) { + this.clusterService = clusterService; + return this; + } + + ClientTransformIndexerBuilder setTransformExtension(TransformExtension transformExtension) { + this.transformExtension = transformExtension; + return this; + } + ClientTransformIndexerBuilder setTransformServices(TransformServices transformServices) { this.transformServices = transformServices; return this; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index e56a54a166b39..4b2da731351d7 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -171,6 +171,8 @@ public TransformIndexer( abstract void doGetFieldMappings(ActionListener> fieldMappingsListener); + abstract void doMaybeCreateDestIndex(Map deducedDestIndexMappings, ActionListener listener); + abstract void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener); abstract void refreshDestinationIndex(ActionListener responseListener); @@ -288,7 +290,7 @@ protected void onStart(long now, ActionListener listener) { // On each run, we need to get the total number of docs and reset the count of processed docs // Since multiple checkpoints can be executed in the task while it is running on the same node, we need to gather // the progress here, and not in the executor. - ActionListener configurationReadyListener = ActionListener.wrap(r -> { + ActionListener configurationReadyListener = ActionListener.wrap(unused -> { initializeFunction(); if (initialRun()) { @@ -339,7 +341,16 @@ protected void onStart(long now, ActionListener listener) { // ... otherwise we fall back to index mappings deduced based on source indices this.fieldMappings = deducedDestIndexMappings.get(); } - configurationReadyListener.onResponse(null); + // Since the unattended transform could not have created the destination index yet, we do it here. + // This is important to create the destination index explicitly before indexing first documents. Otherwise, the destination + // index aliases may be missing. + if (destIndexMappings.isEmpty() + && context.getCheckpoint() == 0 + && Boolean.TRUE.equals(transformConfig.getSettings().getUnattended())) { + doMaybeCreateDestIndex(deducedDestIndexMappings.get(), configurationReadyListener); + } else { + configurationReadyListener.onResponse(null); + } }, listener::onFailure); ActionListener reLoadFieldMappingsListener = ActionListener.wrap(updateConfigResponse -> { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java index 6fefdb236c89a..2fc001599c78d 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutor.java @@ -46,6 +46,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; import org.elasticsearch.xpack.transform.Transform; +import org.elasticsearch.xpack.transform.TransformExtension; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex; @@ -75,7 +76,7 @@ public class TransformPersistentTasksExecutor extends PersistentTasksExecutor stateHolder = new SetOnce<>(); @@ -348,7 +352,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable TransformTa TransformInternalIndex.createLatestVersionedIndexIfRequired( clusterService, parentTaskClient, - transformInternalIndexAdditionalSettings, + transformExtension.getTransformInternalIndexAdditionalSettings(), templateCheckListener ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java index ba7c09ed35a6d..43a8f35cfeafe 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java @@ -22,6 +22,8 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -49,6 +51,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; +import org.elasticsearch.xpack.transform.TransformExtension; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService; @@ -129,6 +132,9 @@ public void testPitInjection() throws InterruptedException { final var client = new PitMockClient(threadPool, true); MockClientTransformIndexer indexer = new MockClientTransformIndexer( mock(ThreadPool.class), + mock(ClusterService.class), + mock(IndexNameExpressionResolver.class), + mock(TransformExtension.class), new TransformServices( mock(IndexBasedTransformConfigManager.class), mock(TransformCheckpointService.class), @@ -223,6 +229,9 @@ public void testPitInjectionIfPitNotSupported() throws InterruptedException { final var client = new PitMockClient(threadPool, false); MockClientTransformIndexer indexer = new MockClientTransformIndexer( mock(ThreadPool.class), + mock(ClusterService.class), + mock(IndexNameExpressionResolver.class), + mock(TransformExtension.class), new TransformServices( mock(IndexBasedTransformConfigManager.class), mock(TransformCheckpointService.class), @@ -306,6 +315,9 @@ public void testDisablePit() throws InterruptedException { final var client = new PitMockClient(threadPool, true); MockClientTransformIndexer indexer = new MockClientTransformIndexer( mock(ThreadPool.class), + mock(ClusterService.class), + mock(IndexNameExpressionResolver.class), + mock(TransformExtension.class), new TransformServices( mock(IndexBasedTransformConfigManager.class), mock(TransformCheckpointService.class), @@ -393,6 +405,9 @@ private static class MockClientTransformIndexer extends ClientTransformIndexer { MockClientTransformIndexer( ThreadPool threadPool, + ClusterService clusterService, + IndexNameExpressionResolver indexNameExpressionResolver, + TransformExtension transformExtension, TransformServices transformServices, CheckpointProvider checkpointProvider, AtomicReference initialState, @@ -409,6 +424,9 @@ private static class MockClientTransformIndexer extends ClientTransformIndexer { ) { super( threadPool, + clusterService, + indexNameExpressionResolver, + transformExtension, transformServices, checkpointProvider, initialState, @@ -546,6 +564,9 @@ private ClientTransformIndexer createTestIndexer(ParentTaskAssigningClient clien return new ClientTransformIndexer( mock(ThreadPool.class), + mock(ClusterService.class), + mock(IndexNameExpressionResolver.class), + mock(TransformExtension.class), new TransformServices( mock(IndexBasedTransformConfigManager.class), mock(TransformCheckpointService.class), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index a18c926e21da6..079ab6afe2200 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -20,6 +20,8 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker.Durability; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Settings; @@ -48,6 +50,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; import org.elasticsearch.xpack.transform.Transform; +import org.elasticsearch.xpack.transform.TransformExtension; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService; @@ -109,6 +112,9 @@ static class MockedTransformIndexer extends ClientTransformIndexer { MockedTransformIndexer( ThreadPool threadPool, + ClusterService clusterService, + IndexNameExpressionResolver indexNameExpressionResolver, + TransformExtension transformExtension, String executorName, IndexBasedTransformConfigManager transformsConfigManager, CheckpointProvider checkpointProvider, @@ -124,6 +130,9 @@ static class MockedTransformIndexer extends ClientTransformIndexer { ) { super( threadPool, + clusterService, + indexNameExpressionResolver, + transformExtension, new TransformServices( transformsConfigManager, mock(TransformCheckpointService.class), @@ -1041,6 +1050,9 @@ private MockedTransformIndexer createMockIndexer( }).when(transformConfigManager).getTransformConfiguration(any(), any()); MockedTransformIndexer indexer = new MockedTransformIndexer( threadPool, + mock(ClusterService.class), + mock(IndexNameExpressionResolver.class), + mock(TransformExtension.class), executorName, transformConfigManager, mock(CheckpointProvider.class), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java index 750e535c4330f..7d5c5a41e3154 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -32,6 +34,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; +import org.elasticsearch.xpack.transform.TransformExtension; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService; @@ -61,6 +64,9 @@ private static class MockClientTransformIndexer extends ClientTransformIndexer { MockClientTransformIndexer( ThreadPool threadPool, + ClusterService clusterService, + IndexNameExpressionResolver indexNameExpressionResolver, + TransformExtension transformExtension, TransformServices transformServices, CheckpointProvider checkpointProvider, AtomicReference initialState, @@ -77,6 +83,9 @@ private static class MockClientTransformIndexer extends ClientTransformIndexer { ) { super( threadPool, + clusterService, + indexNameExpressionResolver, + transformExtension, transformServices, checkpointProvider, initialState, @@ -214,6 +223,9 @@ public void fail(Throwable exception, String failureMessage, ActionListener> fieldMappingsListene fieldMappingsListener.onResponse(Collections.emptyMap()); } + @Override + void doMaybeCreateDestIndex(Map deducedDestIndexMappings, ActionListener listener) { + listener.onResponse(null); + } + @Override void persistState(TransformState state, ActionListener listener) { persistedState = state; @@ -312,6 +317,11 @@ void doGetFieldMappings(ActionListener> fieldMappingsListene fieldMappingsListener.onResponse(Collections.emptyMap()); } + @Override + void doMaybeCreateDestIndex(Map deducedDestIndexMappings, ActionListener listener) { + listener.onResponse(null); + } + @Override void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { responseListener.onResponse( diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index ee86f2ca6fcf4..279ce65be0be2 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -255,6 +255,11 @@ void doGetFieldMappings(ActionListener> fieldMappingsListene fieldMappingsListener.onResponse(Collections.emptyMap()); } + @Override + void doMaybeCreateDestIndex(Map deducedDestIndexMappings, ActionListener listener) { + listener.onResponse(null); + } + public boolean waitingForNextSearch() { return super.getScheduledNextSearch() != null; } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java index 69d81c85a62d3..b32ec235fcc6f 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.core.transform.TransformConfigVersion; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; +import org.elasticsearch.xpack.transform.DefaultTransformExtension; import org.elasticsearch.xpack.transform.Transform; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService; @@ -458,7 +459,7 @@ private TransformPersistentTasksExecutor buildTaskExecutor() { threadPool, clusterService, Settings.EMPTY, - Settings.EMPTY, + new DefaultTransformExtension(), TestIndexNameExpressionResolver.newInstance() ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java index 0e0d88dee7333..a34d35e4d3cb5 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; @@ -43,6 +44,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; +import org.elasticsearch.xpack.transform.DefaultTransformExtension; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.checkpoint.TransformCheckpointService; import org.elasticsearch.xpack.transform.notifications.MockTransformAuditor; @@ -196,6 +198,9 @@ private TransformServices transformServices(Clock clock, TransformAuditor audito private ClientTransformIndexerBuilder indexerBuilder(TransformConfig transformConfig, TransformServices transformServices) { return new ClientTransformIndexerBuilder().setClient(new ParentTaskAssigningClient(client, TaskId.EMPTY_TASK_ID)) + .setClusterService(mock(ClusterService.class)) + .setIndexNameExpressionResolver(mock(IndexNameExpressionResolver.class)) + .setTransformExtension(new DefaultTransformExtension()) .setTransformConfig(transformConfig) .setTransformServices(transformServices); } From ef9609b33433ae9c06094e85638a9234d136ee7d Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 15 Feb 2024 08:49:18 +0100 Subject: [PATCH 46/78] Reduce InternalBinaryRange and InternalRandomSampler in a streaming fashion (#105381) --- .../bucket/range/InternalBinaryRange.java | 54 ++++++------------- .../sampler/random/InternalRandomSampler.java | 10 ++-- 2 files changed, 19 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java index f81a5ffbcaf18..7e02be2581038 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java @@ -18,12 +18,11 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; +import org.elasticsearch.search.aggregations.bucket.FixedMultiBucketAggregatorsReducer; import org.elasticsearch.search.aggregations.support.SamplingContext; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -248,50 +247,27 @@ protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceCont return new AggregatorReducer() { - final List aggregations = new ArrayList<>(size); + final FixedMultiBucketAggregatorsReducer reducer = new FixedMultiBucketAggregatorsReducer<>( + reduceContext, + size, + getBuckets() + ) { + + @Override + protected Bucket createBucket(Bucket proto, long docCount, InternalAggregations aggregations) { + return new Bucket(proto.format, proto.keyed, proto.key, proto.from, proto.to, docCount, aggregations); + } + }; @Override public void accept(InternalAggregation aggregation) { - aggregations.add((InternalBinaryRange) aggregation); + InternalBinaryRange binaryRange = (InternalBinaryRange) aggregation; + reducer.accept(binaryRange.getBuckets()); } @Override public InternalAggregation get() { - reduceContext.consumeBucketsAndMaybeBreak(buckets.size()); - long[] docCounts = new long[buckets.size()]; - InternalAggregations[][] aggs = new InternalAggregations[buckets.size()][]; - for (int i = 0; i < aggs.length; ++i) { - aggs[i] = new InternalAggregations[aggregations.size()]; - } - for (int i = 0; i < aggregations.size(); ++i) { - InternalBinaryRange range = aggregations.get(i); - if (range.buckets.size() != buckets.size()) { - throw new IllegalStateException( - "Expected [" + buckets.size() + "] buckets, but got [" + range.buckets.size() + "]" - ); - } - for (int j = 0; j < buckets.size(); ++j) { - Bucket bucket = range.buckets.get(j); - docCounts[j] += bucket.docCount; - aggs[j][i] = bucket.aggregations; - } - } - List buckets = new ArrayList<>(getBuckets().size()); - for (int i = 0; i < getBuckets().size(); ++i) { - Bucket b = getBuckets().get(i); - buckets.add( - new Bucket( - format, - keyed, - b.key, - b.from, - b.to, - docCounts[i], - InternalAggregations.reduce(Arrays.asList(aggs[i]), reduceContext) - ) - ); - } - return new InternalBinaryRange(name, format, keyed, buckets, metadata); + return new InternalBinaryRange(name, format, keyed, reducer.get(), metadata); } }; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java index c17056cecc053..50b6d8704f0f2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorReducer; +import org.elasticsearch.search.aggregations.AggregatorsReducer; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; @@ -20,8 +21,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; public class InternalRandomSampler extends InternalSingleBucketAggregation implements Sampler { @@ -79,22 +78,21 @@ protected InternalSingleBucketAggregation newAggregation(String name, long docCo protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceContext, int size) { return new AggregatorReducer() { long docCount = 0L; - final List subAggregationsList = new ArrayList<>(size); + final AggregatorsReducer subAggregatorReducer = new AggregatorsReducer(reduceContext, size); @Override public void accept(InternalAggregation aggregation) { docCount += ((InternalSingleBucketAggregation) aggregation).getDocCount(); - subAggregationsList.add(((InternalSingleBucketAggregation) aggregation).getAggregations()); + subAggregatorReducer.accept(((InternalSingleBucketAggregation) aggregation).getAggregations()); } @Override public InternalAggregation get() { - InternalAggregations aggs = InternalAggregations.reduce(subAggregationsList, reduceContext); + InternalAggregations aggs = subAggregatorReducer.get(); if (reduceContext.isFinalReduce() && aggs != null) { SamplingContext context = buildContext(); aggs = InternalAggregations.from(aggs.asList().stream().map(agg -> agg.finalizeSampling(context)).toList()); } - return newAggregation(getName(), docCount, aggs); } }; From fd17e0c8aa43679a58c324e9eb156a72c690d811 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 15 Feb 2024 08:49:39 +0100 Subject: [PATCH 47/78] Reduce InternalMatrixStats in a streaming fashion (#105389) --- .../metric/InternalMatrixStats.java | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java index b7c8e3e4e1c29..fa409c0660166 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java @@ -16,7 +16,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -237,41 +236,38 @@ public Object getProperty(List path) { @Override protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceContext, int size) { - final List aggregations = new ArrayList<>(size); return new AggregatorReducer() { + private RunningStats runningStats; + @Override public void accept(InternalAggregation aggregation) { - // TODO: probably can be done in without collecting the aggregators final InternalMatrixStats internalMatrixStats = (InternalMatrixStats) aggregation; if (internalMatrixStats.stats != null) { - aggregations.add(internalMatrixStats); - } - } - - @Override - public InternalAggregation get() { - // return empty result iff all stats are null - if (aggregations.isEmpty()) { - return new InternalMatrixStats(name, 0, null, new MatrixStatsResults(), getMetadata()); - } - - RunningStats runningStats = new RunningStats(); - for (InternalMatrixStats agg : aggregations) { - final Set missingFields = runningStats.missingFieldNames(agg.stats); + if (runningStats == null) { + runningStats = new RunningStats(); + } + final Set missingFields = runningStats.missingFieldNames(internalMatrixStats.stats); if (missingFields.isEmpty() == false) { throw new IllegalArgumentException( "Aggregation [" - + agg.getName() + + internalMatrixStats.getName() + "] all fields must exist in all indices, but some indices are missing these fields [" + String.join(", ", new TreeSet<>(missingFields)) + "]" ); } - runningStats.merge(agg.stats); + runningStats.merge(internalMatrixStats.stats); } + } + @Override + public InternalAggregation get() { + // return empty result iff all stats are null + if (runningStats == null) { + return new InternalMatrixStats(name, 0, null, new MatrixStatsResults(), getMetadata()); + } if (reduceContext.isFinalReduce()) { - MatrixStatsResults matrixStatsResults = new MatrixStatsResults(runningStats); + final MatrixStatsResults matrixStatsResults = new MatrixStatsResults(runningStats); return new InternalMatrixStats(name, matrixStatsResults.getDocCount(), runningStats, matrixStatsResults, getMetadata()); } return new InternalMatrixStats(name, runningStats.docCount, runningStats, null, getMetadata()); From b552b43abb2508cc52562f5e0185746edc972e1d Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 15 Feb 2024 09:02:17 +0100 Subject: [PATCH 48/78] [Connectors API] Add two null check tests and fix test names (#105507) --- ...pdateConnectorSyncJobErrorActionTests.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionTests.java index 0899aa3b599df..f2bdbaeceb49e 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionTests.java @@ -25,7 +25,7 @@ public void testValidate_WhenConnectorSyncJobIdAndErrorArePresent_ExpectNoValida assertThat(exception, nullValue()); } - public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExceptionValidationError() { + public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { UpdateConnectorSyncJobErrorAction.Request request = new UpdateConnectorSyncJobErrorAction.Request( "", randomAlphaOfLengthBetween(10, 100) @@ -36,7 +36,18 @@ public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExceptionValidationError( assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); } - public void testValidate_WhenErrorIsEmpty_ExceptionValidationError() { + public void testValidate_WhenConnectorSyncJobIdIsNull_ExpectValidationError() { + UpdateConnectorSyncJobErrorAction.Request request = new UpdateConnectorSyncJobErrorAction.Request( + null, + randomAlphaOfLengthBetween(10, 100) + ); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } + + public void testValidate_WhenErrorIsEmpty_ExpectValidationError() { UpdateConnectorSyncJobErrorAction.Request request = new UpdateConnectorSyncJobErrorAction.Request(randomAlphaOfLength(10), ""); ActionRequestValidationException exception = request.validate(); @@ -44,4 +55,11 @@ public void testValidate_WhenErrorIsEmpty_ExceptionValidationError() { assertThat(exception.getMessage(), containsString(UpdateConnectorSyncJobErrorAction.ERROR_EMPTY_MESSAGE)); } + public void testValidate_WhenErrorIsNull_ExpectValidationError() { + UpdateConnectorSyncJobErrorAction.Request request = new UpdateConnectorSyncJobErrorAction.Request(randomAlphaOfLength(10), null); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(UpdateConnectorSyncJobErrorAction.ERROR_EMPTY_MESSAGE)); + } } From 72640731abff4d41685fee836c232f6e95a16cbb Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 15 Feb 2024 09:04:01 +0100 Subject: [PATCH 49/78] [Connectors API] Unify class docs of the connector and sync job state machines (#105511) --- .../ConnectorInvalidStatusTransitionException.java | 6 +++--- .../application/connector/ConnectorStateMachine.java | 10 +++++----- .../syncjob/ConnectorSyncJobStateMachine.java | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorInvalidStatusTransitionException.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorInvalidStatusTransitionException.java index dd863d07a9e90..a05e16eb5f4fa 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorInvalidStatusTransitionException.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorInvalidStatusTransitionException.java @@ -10,10 +10,10 @@ public class ConnectorInvalidStatusTransitionException extends Exception { /** - * Constructs an ConnectorInvalidStatusTransitionException exception with a detailed message. + * Constructs a {@link ConnectorInvalidStatusTransitionException} exception with a detailed message. * - * @param current The current state of the connector. - * @param next The attempted next state of the connector. + * @param current The current state of the {@link Connector}. + * @param next The attempted next state of the {@link Connector}. */ public ConnectorInvalidStatusTransitionException(ConnectorStatus current, ConnectorStatus next) { super( diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java index 8e1c4d95d8527..39a12ba334c30 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java @@ -13,9 +13,9 @@ import java.util.Set; /** - * The {@link ConnectorStateMachine} class manages state transitions for connectors + * The {@link ConnectorStateMachine} class manages state transitions for instances of {@link Connector} * in accordance with the Connector Protocol. - * It defines valid transitions between different connector states and provides a method to validate these transitions. + * It defines valid transitions between instances of {@link ConnectorStatus} and provides a method to validate these transitions. */ public class ConnectorStateMachine { @@ -33,10 +33,10 @@ public class ConnectorStateMachine { ); /** - * Checks if a transition from one connector state to another is valid. + * Checks if a transition from one {@link ConnectorStatus} to another is valid. * - * @param current The current state of the connector. - * @param next The proposed next state of the connector. + * @param current The current {@link ConnectorStatus} of the {@link Connector}. + * @param next The proposed next {@link ConnectorStatus} of the {@link Connector}. */ public static boolean isValidTransition(ConnectorStatus current, ConnectorStatus next) { return validNextStates(current).contains(next); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobStateMachine.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobStateMachine.java index 542181ca44d2f..7a7a05bd5e455 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobStateMachine.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobStateMachine.java @@ -15,9 +15,9 @@ import java.util.Set; /** - * The {@link ConnectorSyncJobStateMachine} class manages state transitions for sync jobs + * The {@link ConnectorSyncJobStateMachine} class manages state transitions for instances of {@link ConnectorSyncJob} * in accordance with the Connector Protocol. - * It defines valid transitions between different connector sync job states and provides a method to validate these transitions. + * It defines valid transitions between instances of {@link ConnectorSyncStatus} and provides a method to validate these transitions. */ public class ConnectorSyncJobStateMachine { @@ -39,10 +39,10 @@ public class ConnectorSyncJobStateMachine { ); /** - * Checks if a transition from one connector sync job state to another is valid. + * Checks if a transition from one {@link ConnectorSyncStatus} to another is valid. * - * @param current The current state of the connector sync job. - * @param next The proposed next state of the connector sync job. + * @param current The current {@link ConnectorSyncStatus} of the {@link ConnectorSyncJob}. + * @param next The proposed next {link ConnectorSyncStatus} of the {@link ConnectorSyncJob}. */ public static boolean isValidTransition(ConnectorSyncStatus current, ConnectorSyncStatus next) { return VALID_TRANSITIONS.getOrDefault(current, Collections.emptySet()).contains(next); From 6827f002bb3af3774beecd480b03d1246bd8395d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Thu, 15 Feb 2024 09:29:41 +0100 Subject: [PATCH 50/78] [DOCS] Adds docs to built-in and Eland model support in Inference API (#105500) Co-authored-by: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> --- .../inference/put-inference.asciidoc | 102 ++++++++++++++++-- 1 file changed, 91 insertions(+), 11 deletions(-) diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index dc0f5615bf0eb..5332808d2ce12 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -6,10 +6,12 @@ experimental[] Creates a model to perform an {infer} task. -IMPORTANT: The {infer} APIs enable you to use certain services, such as ELSER, -OpenAI, or Hugging Face, in your cluster. This is not the same feature that you -can use on an ML node with custom {ml} models. If you want to train and use your -own model, use the <>. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in +{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, or +Hugging Face, in your cluster. For built-in models and models uploaded though +Eland, the {infer} APIs offer an alternative way to use and manage trained +models. However, if you do not plan to use the {infer} APIs to use these models +or if you want to use non-NLP models, use the <>. [discrete] @@ -39,6 +41,7 @@ The following services are available through the {infer} API: * ELSER * Hugging Face * OpenAI +* text embedding (for built-in models and models uploaded through Eland) [discrete] @@ -70,13 +73,15 @@ Available services: * `hugging_face`: specify the `text_embedding` task type to use the Hugging Face service. * `openai`: specify the `text_embedding` task type to use the OpenAI service. +* `text_embedding`: specify the `text_embedding` task type to use the E5 +built-in model or text embedding models uploaded by Eland. `service_settings`:: (Required, object) Settings used to install the {infer} model. These settings are specific to the `service` you specified. + -.`service_settings` for `cohere` +.`service_settings` for the `cohere` service [%collapsible%closed] ===== `api_key`::: @@ -106,19 +111,22 @@ https://docs.cohere.com/reference/embed[Cohere docs]. Defaults to `embed-english-v2.0`. ===== + -.`service_settings` for `elser` +.`service_settings` for the `elser` service [%collapsible%closed] ===== `num_allocations`::: (Required, integer) -The number of model allocations to create. +The number of model allocations to create. `num_allocations` must not exceed the +number of available processors per node divided by the `num_threads`. `num_threads`::: (Required, integer) -The number of threads to use by each model allocation. +The number of threads to use by each model allocation. `num_threads` must not +exceed the number of available processors per node divided by the number of +allocations. Must be a power of 2. Max allowed value is 32. ===== + -.`service_settings` for `hugging_face` +.`service_settings` for the `hugging_face` service [%collapsible%closed] ===== `api_key`::: @@ -138,7 +146,7 @@ the same name and the updated API key. The URL endpoint to use for the requests. ===== + -.`service_settings` for `openai` +.`service_settings` for the `openai` service [%collapsible%closed] ===== `api_key`::: @@ -164,13 +172,36 @@ https://platform.openai.com/account/organization[**Settings** > **Organizations* The URL endpoint to use for the requests. Can be changed for testing purposes. Defaults to `https://api.openai.com/v1/embeddings`. ===== ++ +.`service_settings` for the `text_embedding` service +[%collapsible%closed] +===== +`model_id`::: +(Required, string) +The name of the text embedding model to use for the {infer} task. It can be the +ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or +a text embedding model already +{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland]. + +`num_allocations`::: +(Required, integer) +The number of model allocations to create. `num_allocations` must not exceed the +number of available processors per node divided by the `num_threads`. + +`num_threads`::: +(Required, integer) +The number of threads to use by each model allocation. `num_threads` must not +exceed the number of available processors per node divided by the number of +allocations. Must be a power of 2. Max allowed value is 32. +===== + `task_settings`:: (Optional, object) Settings to configure the {infer} task. These settings are specific to the `` you specified. + -.`task_settings` for `text_embedding` +.`task_settings` for the `text_embedding` task type [%collapsible%closed] ===== `input_type`::: @@ -234,6 +265,31 @@ PUT _inference/text_embedding/cohere-embeddings // TEST[skip:TBD] +[discrete] +[[inference-example-e5]] +===== E5 via the text embedding service + +The following example shows how to create an {infer} model called +`my-e5-model` to perform a `text_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/my-e5-model +{ + "service": "text_embedding", + "service_settings": { + "num_allocations": 1, + "num_threads": 1, + "model_id": ".multilingual-e5-small" <1> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The `model_id` must be the ID of one of the built-in E5 models. Valid values +are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`. For +further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation]. + + [discrete] [[inference-example-elser]] ===== ELSER service @@ -304,6 +360,30 @@ endpoint URL. Select the model you want to use on the new endpoint creation page task under the Advanced configuration section. Create the endpoint. Copy the URL after the endpoint initialization has been finished. +[discrete] +[[inference-example-eland]] +===== Models uploaded by Eland via the text embedding service + +The following example shows how to create an {infer} model called +`my-msmarco-minilm-model` to perform a `text_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/my-msmarco-minilm-model +{ + "service": "text_embedding", + "service_settings": { + "num_allocations": 1, + "num_threads": 1, + "model_id": "msmarco-MiniLM-L12-cos-v5" <1> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The `model_id` must be the ID of a text embedding model which has already +been +{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland]. + [discrete] [[inference-example-openai]] From 4ee086e40652863084ec91dd79976a332bade7fa Mon Sep 17 00:00:00 2001 From: florent-leborgne Date: Thu, 15 Feb 2024 09:39:02 +0100 Subject: [PATCH 51/78] [DOCS] [Remote clusters] Reference specific instructions for cloud trust --- docs/reference/modules/remote-clusters.asciidoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc index 8a0feefeaf21f..8dcdfb009dab5 100644 --- a/docs/reference/modules/remote-clusters.asciidoc +++ b/docs/reference/modules/remote-clusters.asciidoc @@ -30,6 +30,11 @@ capabilities, the local and remote cluster must be on the same [discrete] === Add remote clusters +NOTE: The instructions that follow describe how to create a remote connection from a +self-managed cluster. You can also set up {ccs} and {ccr} from an +link:https://www.elastic.co/guide/en/cloud/current/ec-enable-ccs.html[{ess} deployment] +or from an link:https://www.elastic.co/guide/en/cloud-enterprise/current/ece-enable-ccs.html[{ece} deployment]. + To add remote clusters, you can choose between <> and <>. Both security models are compatible From 37946a7edd342442a23f211edeae29eac1ddf4c9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 09:16:17 +0000 Subject: [PATCH 52/78] Upgrade to Netty 4.1.107 (#105517) Mainly to pick up the fix for netty/netty#13816 to enable some more precise testing of HTTP edge cases. --- build-tools-internal/version.properties | 2 +- docs/changelog/105517.yaml | 5 ++ gradle/verification-metadata.xml | 86 ++++++++++++------------- 3 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 docs/changelog/105517.yaml diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index fa73c475e5242..d1d0da4b1c262 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -14,7 +14,7 @@ log4j = 2.19.0 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.10.0 -netty = 4.1.94.Final +netty = 4.1.107.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 diff --git a/docs/changelog/105517.yaml b/docs/changelog/105517.yaml new file mode 100644 index 0000000000000..7cca86d1cff6e --- /dev/null +++ b/docs/changelog/105517.yaml @@ -0,0 +1,5 @@ +pr: 105517 +summary: Upgrade to Netty 4.1.107 +area: Network +type: upgrade +issues: [] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index ae2ec2345d185..d7e4e1c723a24 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1281,14 +1281,19 @@ + + + + + - - - + + + @@ -1296,29 +1301,29 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -1326,9 +1331,9 @@ - - - + + + @@ -1336,14 +1341,14 @@ - - - + + + - - - + + + @@ -1351,14 +1356,14 @@ - - - + + + - - - + + + @@ -1366,26 +1371,21 @@ - - - - - + + + + + - - - - - From 3d6bfb78ed1871b179c4a5aa61be56f688d2bc4f Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 09:17:12 +0000 Subject: [PATCH 53/78] Eagerly initialize `ReleaseVersions` (#105364) `ReleaseVersions` does some nontrivial initialization. It shouldn't fail, but if it does then it should do so early in startup rather than waiting until the class is first used, which may be in a context in which failure is undesirable. This commit adds it to the list of classes that are initialized even before the security manager is installed. Relates #103627 --- .../main/java/org/elasticsearch/bootstrap/Elasticsearch.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 8b24fbf7b0258..960988db67b33 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -17,6 +17,7 @@ import org.apache.lucene.util.VectorUtil; import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ReleaseVersions; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.filesystem.FileSystemNatives; @@ -185,6 +186,8 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { IfConfig.logIfNecessary(); ensureInitialized( + // ReleaseVersions does nontrivial static initialization which should always succeed but load it now (before SM) to be sure + ReleaseVersions.class, // ReferenceDocs class does nontrivial static initialization which should always succeed but load it now (before SM) to be sure ReferenceDocs.class, // The following classes use MethodHandles.lookup during initialization, load them now (before SM) to be sure they succeed From e64cf991a9963c14a562f3333b6f5a4ba3dd6587 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 15 Feb 2024 10:59:46 +0100 Subject: [PATCH 54/78] Revert "Reduce InternalBinaryRange and InternalRandomSampler in a streaming fashion (#105381)" (#105539) This reverts commit ef9609b33433ae9c06094e85638a9234d136ee7d. --- .../bucket/range/InternalBinaryRange.java | 54 +++++++++++++------ .../sampler/random/InternalRandomSampler.java | 10 ++-- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java index 7e02be2581038..f81a5ffbcaf18 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java @@ -18,11 +18,12 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; -import org.elasticsearch.search.aggregations.bucket.FixedMultiBucketAggregatorsReducer; import org.elasticsearch.search.aggregations.support.SamplingContext; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -247,27 +248,50 @@ protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceCont return new AggregatorReducer() { - final FixedMultiBucketAggregatorsReducer reducer = new FixedMultiBucketAggregatorsReducer<>( - reduceContext, - size, - getBuckets() - ) { - - @Override - protected Bucket createBucket(Bucket proto, long docCount, InternalAggregations aggregations) { - return new Bucket(proto.format, proto.keyed, proto.key, proto.from, proto.to, docCount, aggregations); - } - }; + final List aggregations = new ArrayList<>(size); @Override public void accept(InternalAggregation aggregation) { - InternalBinaryRange binaryRange = (InternalBinaryRange) aggregation; - reducer.accept(binaryRange.getBuckets()); + aggregations.add((InternalBinaryRange) aggregation); } @Override public InternalAggregation get() { - return new InternalBinaryRange(name, format, keyed, reducer.get(), metadata); + reduceContext.consumeBucketsAndMaybeBreak(buckets.size()); + long[] docCounts = new long[buckets.size()]; + InternalAggregations[][] aggs = new InternalAggregations[buckets.size()][]; + for (int i = 0; i < aggs.length; ++i) { + aggs[i] = new InternalAggregations[aggregations.size()]; + } + for (int i = 0; i < aggregations.size(); ++i) { + InternalBinaryRange range = aggregations.get(i); + if (range.buckets.size() != buckets.size()) { + throw new IllegalStateException( + "Expected [" + buckets.size() + "] buckets, but got [" + range.buckets.size() + "]" + ); + } + for (int j = 0; j < buckets.size(); ++j) { + Bucket bucket = range.buckets.get(j); + docCounts[j] += bucket.docCount; + aggs[j][i] = bucket.aggregations; + } + } + List buckets = new ArrayList<>(getBuckets().size()); + for (int i = 0; i < getBuckets().size(); ++i) { + Bucket b = getBuckets().get(i); + buckets.add( + new Bucket( + format, + keyed, + b.key, + b.from, + b.to, + docCounts[i], + InternalAggregations.reduce(Arrays.asList(aggs[i]), reduceContext) + ) + ); + } + return new InternalBinaryRange(name, format, keyed, buckets, metadata); } }; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java index 50b6d8704f0f2..c17056cecc053 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorReducer; -import org.elasticsearch.search.aggregations.AggregatorsReducer; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; @@ -21,6 +20,8 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; public class InternalRandomSampler extends InternalSingleBucketAggregation implements Sampler { @@ -78,21 +79,22 @@ protected InternalSingleBucketAggregation newAggregation(String name, long docCo protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceContext, int size) { return new AggregatorReducer() { long docCount = 0L; - final AggregatorsReducer subAggregatorReducer = new AggregatorsReducer(reduceContext, size); + final List subAggregationsList = new ArrayList<>(size); @Override public void accept(InternalAggregation aggregation) { docCount += ((InternalSingleBucketAggregation) aggregation).getDocCount(); - subAggregatorReducer.accept(((InternalSingleBucketAggregation) aggregation).getAggregations()); + subAggregationsList.add(((InternalSingleBucketAggregation) aggregation).getAggregations()); } @Override public InternalAggregation get() { - InternalAggregations aggs = subAggregatorReducer.get(); + InternalAggregations aggs = InternalAggregations.reduce(subAggregationsList, reduceContext); if (reduceContext.isFinalReduce() && aggs != null) { SamplingContext context = buildContext(); aggs = InternalAggregations.from(aggs.asList().stream().map(agg -> agg.finalizeSampling(context)).toList()); } + return newAggregation(getName(), docCount, aggs); } }; From 7817f61175834748b4eafcd16fe7c789f8a34b3c Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 10:27:19 +0000 Subject: [PATCH 55/78] AwaitsFix for #105542 --- .../search/aggregations/bucket/RandomSamplerIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java index 28c186c559dff..ef7a35940c8dd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java @@ -128,6 +128,7 @@ public void testRandomSampler() { ); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105542") public void testRandomSamplerHistogram() { Map sampleMonotonicValue = new HashMap<>(); Map sampleNumericValue = new HashMap<>(); From 384325abbb520455c8c13dc7e81f23ababfcea25 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 10:32:20 +0000 Subject: [PATCH 56/78] AwaitsFix for #105543 --- .../java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index b23c75df6fa4f..ef013749fc896 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -1316,6 +1316,7 @@ public void testStatsNestFields() { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105543") public void testStatsMissingFieldWithStats() { final String node1, node2; if (randomBoolean()) { From ac2e54d56e311d73110c2bfdb16a6e4b6f7766be Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 15 Feb 2024 11:34:07 +0100 Subject: [PATCH 57/78] [Connectors API] Add more null check tests to connector sync job actions (#105509) --- .../xpack/application/connector/ConnectorTestUtils.java | 3 +++ .../action/CancelConnectorSyncJobActionTests.java | 8 ++++++++ .../action/CheckInConnectorSyncJobActionTests.java | 8 ++++++++ .../action/DeleteConnectorSyncJobActionTests.java | 9 +++++++++ .../syncjob/action/GetConnectorSyncJobActionTests.java | 8 ++++++++ 5 files changed, 36 insertions(+) diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java index 3cb254bbba7d8..3e17c33834989 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java @@ -48,6 +48,9 @@ import static org.elasticsearch.test.ESTestCase.randomLongBetween; public final class ConnectorTestUtils { + + public static final String NULL_STRING = null; + public static PutConnectorAction.Request getRandomPutConnectorActionRequest() { return new PutConnectorAction.Request( randomAlphaOfLengthBetween(5, 15), diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionTests.java index 0dd8d452254dc..f3cf0e7d1f121 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.NULL_STRING; import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; @@ -33,4 +34,11 @@ public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { assertThat(exception.getMessage(), containsString(EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); } + public void testValidate_WhenConnectorSyncJobIdIsNull_ExpectValidationError() { + CancelConnectorSyncJobAction.Request requestWithMissingConnectorId = new CancelConnectorSyncJobAction.Request(NULL_STRING); + ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionTests.java index fe5046e42f828..6dca05e28b6e9 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.NULL_STRING; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -33,4 +34,11 @@ public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); } + public void testValidate_WhenConnectorSyncJobIdIsNull_ExpectValidationError() { + CheckInConnectorSyncJobAction.Request requestWithMissingConnectorSyncJobId = new CheckInConnectorSyncJobAction.Request(NULL_STRING); + ActionRequestValidationException exception = requestWithMissingConnectorSyncJobId.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionTests.java index 00dff3e83211b..66d2f42f3f0b2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.NULL_STRING; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -32,4 +33,12 @@ public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); } + public void testValidate_WhenConnectorSyncJobIdIsNull_ExpectValidationError() { + DeleteConnectorSyncJobAction.Request requestWithMissingConnectorId = new DeleteConnectorSyncJobAction.Request(NULL_STRING); + ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } + } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionTests.java index 807f02124f32a..466bd5444294b 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.NULL_STRING; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -33,4 +34,11 @@ public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); } + public void testValidate_WhenConnectorSyncJobIdIsNull_ExpectValidationError() { + GetConnectorSyncJobAction.Request requestWithMissingConnectorId = new GetConnectorSyncJobAction.Request(NULL_STRING); + ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } } From e241a91a4e02bc3dba1af14fb9880b0370b11ba7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 15 Feb 2024 12:02:13 +0100 Subject: [PATCH 58/78] Docs for hot-reloadable remote cluster credentials (#105483) Docs PR to accompany https://github.com/elastic/elasticsearch/pull/103215. Resolves: ES-7625 --- .../cluster/remote-clusters-api-key.asciidoc | 21 +++++++++-------- .../remote-clusters-migration.asciidoc | 20 ++++++++-------- .../cluster/remote-clusters-settings.asciidoc | 5 +++- .../remote-clusters-troubleshooting.asciidoc | 23 ++++++++----------- docs/reference/setup/secure-settings.asciidoc | 1 + 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc b/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc index 73ec6966ec049..b95ebdf143a57 100644 --- a/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc @@ -31,7 +31,7 @@ In this model, cross-cluster operations use <> (remote cluster interface) for communication between clusters. A remote cluster must enable this port for local clusters to connect. Configure Transport Layer Security (TLS) for this port to maximize security (as explained -in <>). +in <>). The local cluster must trust the remote cluster on the remote cluster interface. This means that the local cluster trusts the remote cluster's certificate @@ -65,15 +65,15 @@ information, refer to https://www.elastic.co/subscriptions. ===== On the remote cluster // tag::remote-cluster-steps[] -. Enable the remote cluster server on every node of the remote cluster. In +. Enable the remote cluster server on every node of the remote cluster. In `elasticsearch.yml`: -.. Set <> to +.. Set <> to `true`. .. Configure the bind and publish address for remote cluster server traffic, for example using <>. Without configuring the address, remote cluster traffic may be bound to the local interface, and remote clusters running on other machines can't connect. -.. Optionally, configure the remote server port using +.. Optionally, configure the remote server port using <> (defaults to `9443`). . Next, generate a certificate authority (CA) and a server certificate/key pair. On one of the nodes of the remote cluster, from the directory where {es} has @@ -86,8 +86,8 @@ been installed: ./bin/elasticsearch-certutil ca --pem --out=cross-cluster-ca.zip --pass CA_PASSWORD ---- + -Replace `CA_PASSWORD` with the password you want to use for the CA. You can -remove the `--pass` option and its argument if you are not deploying to a +Replace `CA_PASSWORD` with the password you want to use for the CA. You can +remove the `--pass` option and its argument if you are not deploying to a production environment. .. Unzip the generated `cross-cluster-ca.zip` file. This compressed file @@ -100,7 +100,7 @@ contains the following content: |_ ca.key ---- -.. Generate a certificate and private key pair for the nodes in the remote +.. Generate a certificate and private key pair for the nodes in the remote cluster: + [source,sh] @@ -183,13 +183,16 @@ Replace `ALIAS` with the same name that you will use to create the remote cluste later. When prompted, enter the encoded cross-cluster API key created on the remote cluster earlier. -. Restart the local cluster to load the keystore change. +. Restart the local cluster to load changes to the keystore and settings. + +**Note:** If you are configuring only the cross-cluster API key, you can call the <> API, instead of restarting the cluster. +Configuring the `remote_cluster_client` settings in `elasticsearch.yml` still requires a restart. [[remote-clusters-connect-api-key]] ==== Connect to a remote cluster :trust-mechanism: api-key include::remote-clusters-connect.asciidoc[] -:!trust-mechanism: +:!trust-mechanism: include::{es-repo-dir}/security/authentication/remote-clusters-privileges-api-key.asciidoc[leveloffset=+1] diff --git a/docs/reference/modules/cluster/remote-clusters-migration.asciidoc b/docs/reference/modules/cluster/remote-clusters-migration.asciidoc index 9db7c4a0257ad..e205d7cb141fe 100644 --- a/docs/reference/modules/cluster/remote-clusters-migration.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-migration.asciidoc @@ -51,7 +51,7 @@ include::remote-clusters-api-key.asciidoc[tag=remote-cluster-steps] [[remote-clusters-migration-stop]] ==== Stop cross-cluster operations -On the local cluster, stop any persistent tasks that refer to the remote +On the local cluster, stop any persistent tasks that refer to the remote cluster: * Use the <> API to stop any transforms. @@ -74,13 +74,13 @@ roles used for cross-cluster operations. You should be able to copy these privileges from the original roles on the remote cluster, where they are defined under the certification based security model. ** The roles on the local cluster can't exceed the `access` privilege granted by -the cross-cluster API key. Any extra local privileges will be suppressed by the +the cross-cluster API key. Any extra local privileges will be suppressed by the cross-cluster API key's privileges. ** No update is needed if the {ccr} or {ccs} tasks have been configured with a `superuser` role. The `superuser` role is automatically updated to allow access to all remote indices. ** Tasks that are run as regular users with named roles are immediately updated -with the new privileges. A task will load a new definition the next time it +with the new privileges. A task will load a new definition the next time it runs. ** You need to restart tasks that are run using an API key (done in a later step). @@ -123,7 +123,7 @@ created on the remote cluster earlier. . If you've dynamically configured the remote cluster (via the cluster settings API): -.. Restart the local cluster to load changes to the keystore. +.. Restart the local cluster to load changes to the keystore and settings. .. Re-add the remote cluster. Use the same remote cluster alias, and change the transport port into the remote cluster port. For example: @@ -188,7 +188,7 @@ remote cluster: ---- // TEST[skip:TODO] <1> The remote cluster is connected. -<2> If present, indicates the remote cluster has connected using API key +<2> If present, indicates the remote cluster has connected using API key authentication. [[remote-clusters-migration-resume]] @@ -204,7 +204,7 @@ task will update the task with the updated API key. * Use the <> API to start any transforms. * Use the <> API to open any anomaly detection jobs. * Use the <> API to resume any auto-follow {ccr}. -* Use the <> API to resume any manual {ccr} or +* Use the <> API to resume any manual {ccr} or existing indices that were created from the auto-follow pattern. [[remote-clusters-migration-disable-cert]] @@ -232,8 +232,8 @@ or distributed, is no longer trusted. Another solution is to apply IP filters to the transport interface, blocking traffic from outside the cluster. -. Optionally, delete any roles on the remote cluster that were only used for -cross-cluster operations. These roles are no longer used under the API key based +. Optionally, delete any roles on the remote cluster that were only used for +cross-cluster operations. These roles are no longer used under the API key based security model. [[remote-clusters-migration-rollback]] @@ -252,7 +252,7 @@ the migration. . On each node, remove the `remote_cluster_client.ssl.*` settings from `elasticsearch.yml`. -. Restart the local cluster to apply changes to the keystore and +. Restart the local cluster to apply changes to the keystore and `elasticsearch.yml`. . On the local cluster, apply the original remote cluster settings. If the @@ -263,4 +263,4 @@ remote cluster connection has been configured statically (using the local cluster has connected to the remote cluster. The response should have `"connected": true` and not have `"cluster_credentials": "::es_redacted::"`. -. Restart any persistent tasks that you've stopped earlier. \ No newline at end of file +. Restart any persistent tasks that you've stopped earlier. diff --git a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc index 85e63918e6fa9..bba8c7ffb3491 100644 --- a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc @@ -65,7 +65,8 @@ mode are described separately. is used as the fallback setting. -`cluster.remote..credentials` (<>):: +`cluster.remote..credentials` (<>, <>):: +[[remote-cluster-credentials-setting]] beta:[] Per cluster setting for configuring <>. @@ -75,6 +76,8 @@ beta:[] The presence (or not) of this setting determines which model a remote cluster uses. If present, the remote cluster uses the API key based model. Otherwise, it uses the certificate based model. + If the setting is added, removed, or updated in the <> and reloaded via the + <> API, the cluster will automatically rebuild its connection to the remote. [[remote-cluster-sniff-settings]] ==== Sniff mode remote cluster settings diff --git a/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc b/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc index 5dc6ab8c08c88..f7b08b40bb7ef 100644 --- a/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc @@ -31,13 +31,13 @@ incoming cross-cluster requests by default, while it is ready to send outgoing cross-cluster requests. Ensure you've enabled the remote cluster server on every node of the remote cluster. In `elasticsearch.yml`: -* Set <> to +* Set <> to `true`. * Configure the bind and publish address for remote cluster server traffic, for example using <>. Without configuring the address, remote cluster traffic may be bound to the local interface, and remote clusters running on other machines can't connect. -* Optionally, configure the remote server port using +* Optionally, configure the remote server port using <> (defaults to `9443`). [[remote-clusters-troubleshooting-common-issues]] @@ -73,7 +73,7 @@ org.elasticsearch.transport.ConnectTransportException: [][192.168.0.42:9443] *co ====== Resolution * Check the host and port for the remote cluster are correct. -* Ensure the <> on the remote cluster. * Ensure no firewall is blocking the communication. @@ -108,11 +108,11 @@ cause of the failure. For example: * Is the remote cluster certificate not signed by a trusted CA? This is the most likely cause. -* Is hostname verification failing? +* Is hostname verification failing? * Is the certificate expired? Once you know the cause, you should be able to fix it by adjusting the remote -cluster related SSL settings on either the local cluster or the remote cluster. +cluster related SSL settings on either the local cluster or the remote cluster. Often, the issue is on the local cluster. For example, fix it by configuring necessary trusted CAs (`xpack.security.remote_cluster_client.ssl.certificate_authorities`). @@ -275,7 +275,7 @@ This does not show up in the logs of the remote cluster. ====== Resolution Add the cross-cluster API key to {es} keystore on every node of the local -cluster. Restart the local cluster to reload the keystore. +cluster. Use the <> API to reload the keystore. [[remote-clusters-troubleshooting-wrong-api-key-type]] ===== Using the wrong API key type @@ -302,8 +302,7 @@ This does not show up in the logs of the remote cluster. Ask the remote cluster administrator to create and distribute a <>. Replace the existing API key in the {es} keystore with this cross-cluster API key on every -node of the local cluster. Restart the local cluster for keystore changes to -take effect. +node of the local cluster. Use the <> API to reload the keystore. [[remote-clusters-troubleshooting-non-valid-api-key]] ===== Invalid API key @@ -334,15 +333,14 @@ The remote cluster logs `Authentication using apikey failed`: Ask the remote cluster administrator to create and distribute a <>. Replace the existing API key in the {es} keystore with this cross-cluster API key on every -node of the local cluster. Restart the local cluster for keystore changes to -take effect. +node of the local cluster. Use the <> API to reload the keystore. [[remote-clusters-troubleshooting-insufficient-privileges]] ===== API key or local user has insufficient privileges The effective permission for a local user running requests on a remote cluster is determined by the intersection of the cross-cluster API key's privileges and -the local user's `remote_indices` privileges. +the local user's `remote_indices` privileges. ====== Symptom @@ -366,8 +364,7 @@ This does not show up in any logs. create and distribute a <>. Replace the existing API key in the {es} keystore with this cross-cluster API key on every -node of the local cluster. Restart the local cluster for keystore changes to -take effect. +node of the local cluster. Use the <> API to reload the keystore. [[remote-clusters-troubleshooting-no-remote_indices-privileges]] ===== Local user has no `remote_indices` privileges diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc index 028e766d063df..cb88be94e17b6 100644 --- a/docs/reference/setup/secure-settings.asciidoc +++ b/docs/reference/setup/secure-settings.asciidoc @@ -64,3 +64,4 @@ There are reloadable secure settings for: * <> * <> * <> +* <> From ec12dc6f3f724146dc7775a357f8972a73ae2896 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 11:37:57 +0000 Subject: [PATCH 59/78] Improve MISSING shards snapshot message (#105478) Today if you try and take a non-partial snapshot of some indices whose primaries are not all assigned you get a failure message saying Indices don't have primary shards This is fairly confusing and suggests that the user might need to try and add primary shards like they can add replicas. In fact the indices _do_ have primary shards, they're just unassigned. This commit clarifies the message, and adds a link to the relevant troubleshooting docs. --- .../DedicatedClusterSnapshotRestoreIT.java | 2 +- .../SharedClusterSnapshotRestoreIT.java | 2 +- .../elasticsearch/common/ReferenceDocs.java | 1 + .../snapshots/SnapshotsService.java | 10 ++- .../common/reference-docs-links.json | 3 +- .../snapshots/SnapshotResiliencyTests.java | 86 ++++++++++++++++++- 6 files changed, 95 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 6ca3fccd1e292..8fbcfe9caafdd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -267,7 +267,7 @@ public void testRestoreIndexWithMissingShards() throws Exception { .setIndices("test-idx-all", "test-idx-none", "test-idx-some", "test-idx-closed") .setWaitForCompletion(true) ); - assertThat(sne.getMessage(), containsString("Indices don't have primary shards")); + assertThat(sne.getMessage(), containsString("the following indices have unassigned primary shards")); if (randomBoolean()) { logger.info("checking snapshot completion using status"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index c13891728f315..e97840341fea5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -769,7 +769,7 @@ public void testUnallocatedShards() { SnapshotException.class, clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx") ); - assertThat(sne.getMessage(), containsString("Indices don't have primary shards")); + assertThat(sne.getMessage(), containsString("the following indices have unassigned primary shards")); assertThat(getRepositoryData("test-repo"), is(RepositoryData.EMPTY)); } diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java index 67a9e23f2297f..4c401ab0ad52c 100644 --- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java +++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java @@ -70,6 +70,7 @@ public enum ReferenceDocs { BOOTSTRAP_CHECK_TOKEN_SSL, BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP, CONTACT_SUPPORT, + UNASSIGNED_SHARDS, // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner ; diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index d2c9035acd299..3f8b19d72070b 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -59,6 +59,7 @@ import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -111,6 +112,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; @@ -4017,14 +4019,18 @@ private SnapshotsInProgress createSnapshot( repositoryName ); if (request.partial() == false) { - Set missing = new HashSet<>(); + Set missing = new TreeSet<>(); // sorted for more usable message for (Map.Entry entry : shards.entrySet()) { if (entry.getValue().state() == ShardState.MISSING) { missing.add(entry.getKey().getIndex().getName()); } } if (missing.isEmpty() == false) { - throw new SnapshotException(snapshot, "Indices don't have primary shards " + missing); + throw new SnapshotException(snapshot, Strings.format(""" + the following indices have unassigned primary shards \ + and cannot be included in a snapshot unless [partial] is set to [true]: %s; \ + for help with troubleshooting unassigned shards see %s + """, missing, ReferenceDocs.UNASSIGNED_SHARDS)); } } final var newEntry = SnapshotsInProgress.startedEntry( diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index 46e32300e70fd..fef69aec2f543 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -30,5 +30,6 @@ "BOOTSTRAP_CHECK_TLS": "bootstrap-checks-xpack.html#bootstrap-checks-tls", "BOOTSTRAP_CHECK_TOKEN_SSL": "bootstrap-checks-xpack.html#_token_ssl_check", "BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP": "security-minimal-setup.html", - "CONTACT_SUPPORT": "troubleshooting.html#troubleshooting-contact-support" + "CONTACT_SUPPORT": "troubleshooting.html#troubleshooting-contact-support", + "UNASSIGNED_SHARDS": "red-yellow-cluster-status.html" } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index a06e2ffeeb899..96c5075dd3ca7 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -118,6 +118,7 @@ import org.elasticsearch.cluster.service.FakeThreadPoolMasterService; import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput; @@ -224,11 +225,12 @@ import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; import static org.elasticsearch.monitor.StatusInfo.Status.HEALTHY; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.iterableWithSize; @@ -433,9 +435,15 @@ public void testSnapshotWithNodeDisconnects() { } }, e -> { if (partial == false) { - final SnapshotException unwrapped = (SnapshotException) ExceptionsHelper.unwrap(e, SnapshotException.class); - assertNotNull(unwrapped); - assertThat(unwrapped.getMessage(), endsWith("Indices don't have primary shards [test]")); + assertThat( + asInstanceOf(SnapshotException.class, ExceptionsHelper.unwrap(e, SnapshotException.class)).getMessage(), + allOf( + containsString("the following indices have unassigned primary shards"), + containsString("unless [partial] is set to [true]"), + containsString("[test]"), + containsString(ReferenceDocs.UNASSIGNED_SHARDS.toString()) + ) + ); snapshotNeverStarted.set(true); } else { throw new AssertionError(e); @@ -1378,6 +1386,76 @@ public TransportRequestHandler interceptHandler( safeAwait(testListener); // shouldn't throw } + public void testFullSnapshotUnassignedShards() { + setupTestCluster(1, 0); // no data nodes, we want unassigned shards + + final var indices = IntStream.range(0, between(1, 4)).mapToObj(i -> "index-" + i).sorted().toList(); + final var repoName = "repo"; + final var originalSnapshotName = "original-snapshot"; + + var testListener = SubscribableListener + + // Create the repo and indices + .newForked(stepListener -> { + try (var listeners = new RefCountingListener(stepListener)) { + client().admin() + .cluster() + .preparePutRepository(repoName) + .setType(FsRepository.TYPE) + .setSettings(Settings.builder().put("location", randomAlphaOfLength(10))) + .execute(listeners.acquire(createRepoResponse -> {})); + + for (final var index : indices) { + deterministicTaskQueue.scheduleNow( + // wrapped in another scheduleNow() to randomize creation order + ActionRunnable.wrap( + listeners.acquire(createIndexResponse -> {}), + l -> client().admin() + .indices() + .create( + new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.NONE) + .settings(defaultIndexSettings(1)), + l + ) + ) + ); + } + } + }) + + // Take a full snapshot for use as the source for future clones + .andThen( + (l, ignored) -> client().admin() + .cluster() + .prepareCreateSnapshot(repoName, originalSnapshotName) + .setWaitForCompletion(randomBoolean()) + .execute(new ActionListener<>() { + @Override + public void onResponse(CreateSnapshotResponse createSnapshotResponse) { + fail("snapshot should not have started"); + } + + @Override + public void onFailure(Exception e) { + assertThat( + asInstanceOf(SnapshotException.class, e).getMessage(), + allOf( + containsString("the following indices have unassigned primary shards"), + containsString("unless [partial] is set to [true]"), + containsString(indices.toString() /* NB sorted */), + containsString(ReferenceDocs.UNASSIGNED_SHARDS.toString()) + ) + ); + l.onResponse(null); + } + }) + ); + + deterministicTaskQueue.runAllRunnableTasks(); + assertTrue("executed all runnable tasks but test steps are still incomplete", testListener.isDone()); + safeAwait(testListener); // shouldn't throw + } + private RepositoryData getRepositoryData(Repository repository) { final PlainActionFuture res = new PlainActionFuture<>(); repository.getRepositoryData(deterministicTaskQueue::scheduleNow, res); From 2caf17cc6640236200fea3c6ed49cc51f2dbf582 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 15 Feb 2024 11:38:16 +0000 Subject: [PATCH 60/78] Update min CCS version to that used by 8.12 (#104739) --- server/src/main/java/org/elasticsearch/TransportVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 6ac20c1753664..a6fa7a9ea8e99 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -197,7 +197,7 @@ static TransportVersion def(int id) { * Reference to the minimum transport version that can be used with CCS. * This should be the transport version used by the previous minor release. */ - public static final TransportVersion MINIMUM_CCS_VERSION = V_8_11_X; + public static final TransportVersion MINIMUM_CCS_VERSION = V_8_12_0; static final NavigableMap VERSION_IDS = getAllVersionIds(TransportVersions.class); From fb71c6d601a1355a35fc77860d35f5bfec32227d Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 11:40:16 +0000 Subject: [PATCH 61/78] Remove unnecessary BwC in StatelessPrimaryRelocationAction (#105538) There are no remaining clusters using this action with the pre-8.11.x transport protocol so we can remove the BwC here. --- .../recovery/StatelessPrimaryRelocationAction.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java b/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java index 583a38a9da41c..bdc7f5b2aafce 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java @@ -8,7 +8,6 @@ package org.elasticsearch.indices.recovery; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; @@ -49,11 +48,7 @@ public Request(StreamInput in) throws IOException { shardId = new ShardId(in); targetNode = new DiscoveryNode(in); targetAllocationId = in.readString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_11_X)) { - clusterStateVersion = in.readVLong(); - } else { - clusterStateVersion = 0L; // temporary bwc: do not wait for cluster state to be applied - } + clusterStateVersion = in.readVLong(); } @Override @@ -68,9 +63,7 @@ public void writeTo(StreamOutput out) throws IOException { shardId.writeTo(out); targetNode.writeTo(out); out.writeString(targetAllocationId); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_11_X)) { - out.writeVLong(clusterStateVersion); - } // temporary bwc: just omit it, the receiver doesn't wait for a cluster state anyway + out.writeVLong(clusterStateVersion); } public long recoveryId() { From 4faaa284994c02a3b34e4c6f8406e04ddedccb2b Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 15 Feb 2024 13:05:02 +0100 Subject: [PATCH 62/78] Unmute RandomSamplerIT#testRandomSamplerHistogram (#105548) The faulty commit was reverted in e64cf991a9963c14a562f3333b6f5a4ba3dd6587 fixes https://github.com/elastic/elasticsearch/issues/105542 --- .../search/aggregations/bucket/RandomSamplerIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java index ef7a35940c8dd..28c186c559dff 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java @@ -128,7 +128,6 @@ public void testRandomSampler() { ); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105542") public void testRandomSamplerHistogram() { Map sampleMonotonicValue = new HashMap<>(); Map sampleNumericValue = new HashMap<>(); From 949a294ec9c53d9e13c7b6d509ff2897935c1d09 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Thu, 15 Feb 2024 08:35:32 -0500 Subject: [PATCH 63/78] Resolve/cluster should handle IllegalArgumentException 'Unable to open connection' as connected=false and not display error (#105512) Fixed https://github.com/elastic/elasticsearch/issues/105489 --- .../org/elasticsearch/indices/cluster/ResolveClusterIT.java | 1 - .../admin/indices/resolve/TransportResolveClusterAction.java | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java index eda16beb33198..c4be9568f8bab 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java @@ -522,7 +522,6 @@ public void testClusterResolveWithMatchingAliases() throws IOException { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105489") public void testClusterResolveDisconnectedAndErrorScenarios() throws Exception { Map testClusterInfo = setupThreeClusters(false); String localIndex = (String) testClusterInfo.get("local.index"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java index dbb51986dded3..b3f2015b9f5ae 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java @@ -253,6 +253,9 @@ private boolean notConnectedError(Exception e) { if (e instanceof ConnectTransportException || e instanceof NoSuchRemoteClusterException) { return true; } + if (e instanceof IllegalStateException && e.getMessage().contains("Unable to open any connections")) { + return true; + } Throwable ill = ExceptionsHelper.unwrap(e, IllegalArgumentException.class); if (ill != null && ill.getMessage().contains("unknown host")) { return true; From ce265438b2608073996b4ac7b0a2374f26aa4fcb Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 15 Feb 2024 14:46:23 +0100 Subject: [PATCH 64/78] Reduce InternalIpPrefix in a streaming fashion (#105479) --- .../bucket/prefix/InternalIpPrefix.java | 94 ++++++------------- 1 file changed, 28 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefix.java index 5e4a6219da229..a3f53b494acfa 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefix.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefix.java @@ -9,9 +9,9 @@ package org.elasticsearch.search.aggregations.bucket.prefix; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.PriorityQueue; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorReducer; @@ -19,12 +19,14 @@ import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; import org.elasticsearch.search.aggregations.KeyComparable; -import org.elasticsearch.search.aggregations.bucket.IteratorAndCurrent; +import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregatorsReducer; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -223,75 +225,46 @@ protected void doWriteTo(StreamOutput out) throws IOException { @Override protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceContext, int size) { return new AggregatorReducer() { - final List aggregations = new ArrayList<>(size); + final Map buckets = new HashMap<>(); @Override public void accept(InternalAggregation aggregation) { - aggregations.add((InternalIpPrefix) aggregation); + final InternalIpPrefix ipPrefix = (InternalIpPrefix) aggregation; + for (Bucket bucket : ipPrefix.getBuckets()) { + ReducerAndProto reducerAndProto = buckets.computeIfAbsent( + bucket.key, + k -> new ReducerAndProto(new MultiBucketAggregatorsReducer(reduceContext, size), bucket) + ); + reducerAndProto.reducer.accept(bucket); + } } @Override public InternalAggregation get() { - List reducedBuckets = reduceBuckets(aggregations, reduceContext); + final List reducedBuckets = new ArrayList<>(buckets.size()); + for (ReducerAndProto reducerAndProto : buckets.values()) { + if (false == reduceContext.isFinalReduce() || reducerAndProto.reducer.getDocCount() >= minDocCount) { + reducedBuckets.add( + createBucket(reducerAndProto.proto, reducerAndProto.reducer.get(), reducerAndProto.reducer.getDocCount()) + ); + } + } reduceContext.consumeBucketsAndMaybeBreak(reducedBuckets.size()); + reducedBuckets.sort(Comparator.comparing(a -> a.key)); return new InternalIpPrefix(getName(), format, keyed, minDocCount, reducedBuckets, metadata); } - }; - } - private List reduceBuckets(List aggregations, AggregationReduceContext reduceContext) { - final PriorityQueue> pq = new PriorityQueue<>(aggregations.size()) { @Override - protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) { - return a.current().key.compareTo(b.current().key) < 0; - } - }; - for (InternalIpPrefix ipPrefix : aggregations) { - if (ipPrefix.buckets.isEmpty() == false) { - pq.add(new IteratorAndCurrent<>(ipPrefix.buckets.iterator())); - } - } - - List reducedBuckets = new ArrayList<>(); - if (pq.size() > 0) { - // list of buckets coming from different shards that have the same value - List currentBuckets = new ArrayList<>(); - BytesRef value = pq.top().current().key; - - do { - final IteratorAndCurrent top = pq.top(); - if (top.current().key.equals(value) == false) { - final Bucket reduced = reduceBucket(currentBuckets, reduceContext); - if (false == reduceContext.isFinalReduce() || reduced.getDocCount() >= minDocCount) { - reducedBuckets.add(reduced); - } - currentBuckets.clear(); - value = top.current().key; - } - - currentBuckets.add(top.current()); - - if (top.hasNext()) { - top.next(); - assert top.current().key.compareTo(value) > 0 - : "shards must return data sorted by value [" + top.current().key + "] and [" + value + "]"; - pq.updateTop(); - } else { - pq.pop(); - } - } while (pq.size() > 0); - - if (currentBuckets.isEmpty() == false) { - final Bucket reduced = reduceBucket(currentBuckets, reduceContext); - if (false == reduceContext.isFinalReduce() || reduced.getDocCount() >= minDocCount) { - reducedBuckets.add(reduced); + public void close() { + for (ReducerAndProto reducerAndProto : buckets.values()) { + Releasables.close(reducerAndProto.reducer); } } - } - - return reducedBuckets; + }; } + private record ReducerAndProto(MultiBucketAggregatorsReducer reducer, Bucket proto) {} + @Override public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { if (keyed) { @@ -342,17 +315,6 @@ private Bucket createBucket(Bucket prototype, InternalAggregations aggregations, ); } - private Bucket reduceBucket(List buckets, AggregationReduceContext context) { - assert buckets.isEmpty() == false; - long docCount = 0; - for (InternalIpPrefix.Bucket bucket : buckets) { - docCount += bucket.docCount; - } - final List aggregations = new BucketAggregationList<>(buckets); - final InternalAggregations aggs = InternalAggregations.reduce(aggregations, context); - return createBucket(buckets.get(0), aggs, docCount); - } - @Override public List getBuckets() { return Collections.unmodifiableList(buckets); From 5b80065dfea4d5655b59dc1b8a43db80a49fc6f1 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Thu, 15 Feb 2024 09:23:48 -0600 Subject: [PATCH 65/78] Update FIPS documentation for 8.x (#105041) This commit updates the documentation for FIPS support. In addition to the changes for 8.x it also provides more details for how to setup/configure FIPS mode. --- .../security/fips-140-compliance.asciidoc | 165 ++++++++++++++---- docs/reference/security/fips-java17.asciidoc | 13 +- 2 files changed, 132 insertions(+), 46 deletions(-) diff --git a/docs/reference/security/fips-140-compliance.asciidoc b/docs/reference/security/fips-140-compliance.asciidoc index 785c720dba407..bf880213c2073 100644 --- a/docs/reference/security/fips-140-compliance.asciidoc +++ b/docs/reference/security/fips-140-compliance.asciidoc @@ -8,59 +8,75 @@ government computer security standard used to approve cryptographic modules. {es} offers a FIPS 140-2 compliant mode and as such can run in a FIPS 140-2 configured JVM. -IMPORTANT: The JVM bundled with {es} is not configured for FIPS 140-2. You must +IMPORTANT: The JVM bundled with {es} is not configured for FIPS 140-2. You must configure an external JDK with a FIPS 140-2 certified Java Security Provider. Refer to the {es} https://www.elastic.co/support/matrix#matrix_jvm[JVM support matrix] for -supported JVM configurations. +supported JVM configurations. See https://www.elastic.co/subscriptions[subscriptions] for required licensing. -After configuring your JVM for FIPS 140-2, you can run {es} in FIPS 140-2 mode by -setting the `xpack.security.fips_mode.enabled` to `true` in `elasticsearch.yml`. +Compliance with FIPS 140-2 requires using only FIPS approved / NIST recommended cryptographic algorithms. Generally this can be done by the following: -For {es}, adherence to FIPS 140-2 is ensured by: - -- Using FIPS approved / NIST recommended cryptographic algorithms. -- Delegating the implementation of these cryptographic algorithms to a NIST - validated cryptographic module (available via the Java Security Provider - in use in the JVM). -- Allowing the configuration of {es} in a FIPS 140-2 compliant manner, as - documented below. +- Installation and configuration of a FIPS certified Java security provider. +- Ensuring the configuration of {es} is FIPS 140-2 compliant as documented below. +- Setting `xpack.security.fips_mode.enabled` to `true` in `elasticsearch.yml`. Note - this setting alone is not sufficient to be compliant +with FIPS 140-2. [discrete] -=== Upgrade considerations +=== Configuring {es} for FIPS 140-2 -[IMPORTANT] -==== -include::fips-java17.asciidoc[] -==== +Detailed instructions for the configuration required for FIPS 140-2 compliance is beyond the scope of this document. It is the responsibility +of the user to ensure compliance with FIPS 140-2. {es} has been tested with a specific configuration described below. However, there are +other configurations possible to achieve compliance. +The following is a high-level overview of the required configuration: -If you plan to upgrade your existing cluster to a version that can be run in -a FIPS 140-2 configured JVM, we recommend to first perform a rolling -upgrade to the new version in your existing JVM and perform all necessary -configuration changes in preparation for running in FIPS 140-2 mode. You can then -perform a rolling restart of the nodes, starting each node in a FIPS 140-2 JVM. -During the restart, {es}: +* Use an externally installed Java installation. The JVM bundled with {es} is not configured for FIPS 140-2. +* Install a FIPS certified security provider .jar file(s) in {es}'s `lib` directory. +* Configure Java to use a FIPS certified security provider (xref:java-security-provider[see below]). +* Configure {es}'s security manager to allow use of the FIPS certified provider (xref:java-security-manager[see below]). +* Ensure the keystore and truststore are configured correctly (xref:keystore-fips-password[see below]). +* Ensure the TLS settings are configured correctly (xref:fips-tls[see below]). +* Ensure the password hashing settings are configured correctly (xref:fips-stored-password-hashing[see below]). +* Ensure the cached password hashing settings are configured correctly (xref:fips-cached-password-hashing[see below]). +* Configure `elasticsearch.yml` to use FIPS 140-2 mode, see (xref:configuring-es-yml[below]). +* Verify the security provider is installed and configured correctly (xref:verify-security-provider[see below]). +* Review the upgrade considerations (xref:fips-upgrade-considerations[see below]) and limitations (xref:fips-limitations[see below]). -- Upgrades <> to the latest, compliant format. - A FIPS 140-2 JVM cannot load previous format versions. If your keystore is - not password-protected, you must manually set a password. See - <>. -- Upgrades self-generated trial licenses to the latest FIPS 140-2 compliant format. -If your {subscriptions}[subscription] already supports FIPS 140-2 mode, you -can elect to perform a rolling upgrade while at the same time running each -upgraded node in a FIPS 140-2 JVM. In this case, you would need to also manually -regenerate your `elasticsearch.keystore` and migrate all secure settings to it, -in addition to the necessary configuration changes outlined below, before -starting each node. +[discrete] +[[java-security-provider]] +==== Java security provider + +Detailed instructions for installation and configuration of a FIPS certified Java security provider is beyond the scope of this document. +Specifically, a FIPS certified +https://docs.oracle.com/en/java/javase/17/security/java-cryptography-architecture-jca-reference-guide.html[JCA] and +https://docs.oracle.com/en/java/javase/17/security/java-secure-socket-extension-jsse-reference-guide.html[JSSE] implementation is required +so that the JVM uses FIPS validated implementations of NIST recommended cryptographic algorithms. + +Elasticsearch has been tested with Bouncy Castle's https://repo1.maven.org/maven2/org/bouncycastle/bc-fips/1.0.2.4/bc-fips-1.0.2.4.jar[bc-fips 1.0.2.4] +and https://repo1.maven.org/maven2/org/bouncycastle/bctls-fips/1.0.17/bctls-fips-1.0.17.jar[bctls-fips 1.0.17]. +Please refer to the [Support Matrix] for details on which combinations of JVM and security provider are supported in FIPS mode. Elasticsearch does not ship with a FIPS certified provider. It is the responsibility of the user +to install and configure the security provider to ensure compliance with FIPS 140-2. Using a FIPS certified provider will ensure that only +approved cryptographic algorithms are used. + +To configure {es} to use additional security provider(s) configure {es}'s <> `java.security.properties` to point to a file +(https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/fips_java.security[example]) in {es}'s +`config` directory. Ensure the FIPS certified security provider is configured with the lowest order. This file should contain the necessary +configuration to instruct Java to use the FIPS certified security provider. [discrete] -=== Configuring {es} for FIPS 140-2 +[[java-security-manager]] +==== Java security manager + +All code running in {es} is subject to the security restrictions enforced by the Java security manager. +The security provider you have installed and configured may require additional permissions in order to function correctly. You can grant these permissions by providing your own +https://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html#FileSyntax[Java security policy] + +To configure {es}'s security manager configure the JVM property `java.security.policy` to point a file +(https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/fips_java.policy[example])in {es}'s +`config` directory with the desired permissions. This file should contain the necessary configuration for the Java security manager +to grant the required permissions needed by the security provider. -Apart from setting `xpack.security.fips_mode.enabled`, a number of security -related settings need to be configured accordingly in order to be compliant -and able to run {es} successfully in a FIPS 140-2 configured JVM. [discrete] [[keystore-fips-password]] @@ -78,6 +94,7 @@ Note that when the keystore is password-protected, you must supply the password Elasticsearch starts. [discrete] +[[fips-tls]] ==== TLS SSLv2 and SSLv3 are not allowed by FIPS 140-2, so `SSLv2Hello` and `SSLv3` cannot @@ -172,6 +189,78 @@ hashes using non-compliant algorithms will be discarded and the new ones will be created using the algorithm you have selected. [discrete] +[[configuring-es-yml]] +==== Configure {es} elasticsearch.yml + +* Set `xpack.security.fips_mode.enabled` to `true` in `elasticsearch.yml`. This setting is used to ensure to configure some internal +configuration to be FIPS 140-2 compliant and provides some additional verification. + +* Set `xpack.security.autoconfiguration.enabled` to `false`. This will disable the automatic configuration of the security settings. +Users must ensure that the security settings are configured correctly for FIPS-140-2 compliance. This is only applicable for new installations. + +* Set `xpack.security.authc.password_hashing.algorithm` appropriately see xref:fips-stored-password-hashing[above]. + +* Other relevant security settings. For example, TLS for the transport and HTTP interfaces. (not explicitly covered here or in the example below) + +* Optional: Set `xpack.security.fips_mode.required_providers` in `elasticsearch.yml` to ensure the required security providers (8.13+). +see xref:verify-security-provider[below]. + +[source,yaml] +-------------------------------------------------- +xpack.security.fips_mode.enabled: true +xpack.security.autoconfiguration.enabled: false +xpack.security.fips_mode.required_providers: ["BCFIPS", "BCJSSE"] +xpack.security.authc.password_hashing.algorithm: "pbkdf2_stretch" +-------------------------------------------------- + +[discrete] +[[verify-security-provider]] +==== Verify the security provider is installed + +To verify that the security provider is installed and in use, you can use any of the following steps: + +* Verify the required security providers are configured with the lowest order in the file pointed to by `java.security.properties`. +For example, `security.provider.1` is a lower order than `security.provider.2` + +* Set `xpack.security.fips_mode.required_providers` in `elasticsearch.yml` to the list of required security providers. +This setting is used to ensure that the correct security provider is installed and configured. (8.13+) +If the security provider is not installed correctly, {es} will fail to start. `["BCFIPS", "BCJSSE"]` are the values to +use for Bouncy Castle's FIPS JCE and JSSE certified provider. + +[discrete] +[[fips-upgrade-considerations]] +=== Upgrade considerations +include::fips-java17.asciidoc[] + +[IMPORTANT] +==== +Some encryption algorithms may no longer be available by default in updated FIPS 140-2 security providers. +Notably, Triple DES and PKCS1.5 RSA are now discouraged and https://www.bouncycastle.org/fips-java[Bouncy Castle] now +requires explicit configuration to continue using these algorithms. +==== + +If you plan to upgrade your existing cluster to a version that can be run in +a FIPS 140-2 configured JVM, we recommend to first perform a rolling +upgrade to the new version in your existing JVM and perform all necessary +configuration changes in preparation for running in FIPS 140-2 mode. You can then +perform a rolling restart of the nodes, starting each node in a FIPS 140-2 JVM. +During the restart, {es}: + +- Upgrades <> to the latest, compliant format. +A FIPS 140-2 JVM cannot load previous format versions. If your keystore is +not password-protected, you must manually set a password. See +<>. +- Upgrades self-generated trial licenses to the latest FIPS 140-2 compliant format. + +If your {subscriptions}[subscription] already supports FIPS 140-2 mode, you +can elect to perform a rolling upgrade while at the same time running each +upgraded node in a FIPS 140-2 JVM. In this case, you would need to also manually +regenerate your `elasticsearch.keystore` and migrate all secure settings to it, +in addition to the necessary configuration changes outlined below, before +starting each node. + +[discrete] +[[fips-limitations]] === Limitations Due to the limitations that FIPS 140-2 compliance enforces, a small number of diff --git a/docs/reference/security/fips-java17.asciidoc b/docs/reference/security/fips-java17.asciidoc index ee1c9bf15eba0..dc46a3ec628f5 100644 --- a/docs/reference/security/fips-java17.asciidoc +++ b/docs/reference/security/fips-java17.asciidoc @@ -1,10 +1,7 @@ -{es} {version} requires Java 17 or later. -There is not yet a FIPS-certified security module for Java 17 -that you can use when running {es} {version} in FIPS 140-2 mode. -If you run in FIPS 140-2 mode, you will either need to request -an exception from your security organization to upgrade to {es} {version}, -or remain on {es} 7.x until Java 17 is certified. -ifeval::["{release-state}"=="released"] +{es} 8.0+ requires Java 17 or later. {es} 8.13+ has been tested with https://www.bouncycastle.org/java.html[Bouncy Castle]'s Java 17 +https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4616[certified] FIPS implementation and is the +recommended Java security provider when running {es} in FIPS 140-2 mode. +Note - {es} does not ship with a FIPS certified security provider and requires explicit installation and configuration. + Alternatively, consider using {ess} in the https://www.elastic.co/industries/public-sector/fedramp[FedRAMP-certified GovCloud region]. -endif::[] \ No newline at end of file From 309adbfc54ddf14406fb6290a6686b8d7a922ba6 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Thu, 15 Feb 2024 10:25:24 -0500 Subject: [PATCH 66/78] Fix bug in rule_query where text_expansion errored because it was not rewritten (#105365) --- docs/changelog/105365.yaml | 6 + x-pack/plugin/ent-search/qa/rest/build.gradle | 2 +- .../test/entsearch/260_rule_query_search.yml | 150 ++++++++++++++++++ .../application/rules/RuleQueryBuilder.java | 66 ++++---- .../rules/RuleQueryBuilderTests.java | 3 +- 5 files changed, 190 insertions(+), 37 deletions(-) create mode 100644 docs/changelog/105365.yaml diff --git a/docs/changelog/105365.yaml b/docs/changelog/105365.yaml new file mode 100644 index 0000000000000..265e6dccc3915 --- /dev/null +++ b/docs/changelog/105365.yaml @@ -0,0 +1,6 @@ +pr: 105365 +summary: Fix bug in `rule_query` where `text_expansion` errored because it was not + rewritten +area: Application +type: bug +issues: [] diff --git a/x-pack/plugin/ent-search/qa/rest/build.gradle b/x-pack/plugin/ent-search/qa/rest/build.gradle index c9b1557d74a9c..37f1d8f13c850 100644 --- a/x-pack/plugin/ent-search/qa/rest/build.gradle +++ b/x-pack/plugin/ent-search/qa/rest/build.gradle @@ -7,7 +7,7 @@ dependencies { restResources { restApi { - include '_common', 'bulk', 'cluster', 'connector', 'nodes', 'indices', 'index', 'query_ruleset', 'search_application', 'xpack', 'security', 'search' + include '_common', 'bulk', 'cluster', 'connector', 'nodes', 'indices', 'index', 'query_ruleset', 'search_application', 'xpack', 'security', 'search', 'ml' } } diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml index c287209da5bed..40cdb7839c9ed 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml @@ -236,4 +236,154 @@ setup: - match: { hits.total.value: 1 } - match: { hits.hits.0._id: 'doc1' } +--- +"Perform a rule query with an organic query that must be rewritten to another query type": + - skip: + version: " - 8.13.99" + reason: Bugfix that was broken in previous versions + + - do: + indices.create: + index: test-index-with-sparse-vector + body: + mappings: + properties: + source_text: + type: keyword + ml.tokens: + type: sparse_vector + + - do: + ml.put_trained_model: + model_id: "text_expansion_model" + body: > + { + "description": "simple model for testing", + "model_type": "pytorch", + "inference_config": { + "text_expansion": { + "tokenization": { + "bert": { + "with_special_tokens": false + } + } + } + } + } + - do: + ml.put_trained_model_vocabulary: + model_id: "text_expansion_model" + body: > + { "vocabulary": ["[PAD]", "[UNK]", "these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"] } + - do: + ml.put_trained_model_definition_part: + model_id: "text_expansion_model" + part: 0 + body: > + { + "total_definition_length":2078, + "definition": "UEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAAUAA4Ac2ltcGxlbW9kZWwvZGF0YS5wa2xGQgoAWlpaWlpaWlpaWoACY19fdG9yY2hfXwpUaW55VGV4dEV4cGFuc2lvbgpxACmBfShYCAAAAHRyYWluaW5ncQGJWBYAAABfaXNfZnVsbF9iYWNrd2FyZF9ob29rcQJOdWJxAy5QSwcIITmbsFgAAABYAAAAUEsDBBQACAgIAAAAAAAAAAAAAAAAAAAAAAAdAB0Ac2ltcGxlbW9kZWwvY29kZS9fX3RvcmNoX18ucHlGQhkAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWoWRT4+cMAzF7/spfASJomF3e0Ga3nrrn8vcELIyxAzRhAQlpjvbT19DWDrdquqBA/bvPT87nVUxwsm41xPd+PNtUi4a77KvXs+W8voBAHFSQY3EFCIiHKFp1+p57vs/ShyUccZdoIaz93aBTMR+thbPqru+qKBx8P4q/e8TyxRlmwVctJp66H1YmCyS7WsZwD50A2L5V7pCBADGTTOj0bGGE7noQyqzv5JDfp0o9fZRCWqP37yjhE4+mqX5X3AdFZHGM/2TzOHDpy1IvQWR+OWo3KwsRiKdpcqg4pBFDtm+QJ7nqwIPckrlnGfFJG0uNhOl38Sjut3pCqg26QuZy8BR9In7ScHHrKkKMW0TIucFrGQXCMpdaDO05O6DpOiy8e4kr0Ed/2YKOIhplW8gPr4ntygrd9ixpx3j9UZZVRagl2c6+imWUzBjuf5m+Ch7afphuvvW+r/0dsfn+2N9MZGb9+/SFtCYdhd83CMYp+mGy0LiKNs8y/eUuEA8B/d2z4dfUEsHCFSE3IaCAQAAIAMAAFBLAwQUAAgICAAAAAAAAAAAAAAAAAAAAAAAJwApAHNpbXBsZW1vZGVsL2NvZGUvX190b3JjaF9fLnB5LmRlYnVnX3BrbEZCJQBaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpahZHLbtNAFIZtp03rSVIuLRKXjdk5ojitKJsiFq24lem0KKSqpRIZt55gE9/GM+lNLFgx4i1Ys2aHhIBXgAVICNggHgNm6rqJN2BZGv36/v/MOWeea/Z5RVHurLfRUsfZXOnccx522itrd53O0vLqbaKYtsAKUe1pcege7hm9JNtzM8+kOOzNApIX0A3xBXE6YE7g0UWjg2OaZAJXbKvALOnj2GEHKc496ykLktgNt3Jz17hprCUxFqExe7YIpQkNpO1/kfHhPUdtUAdH2/gfmeYiIFW7IkM6IBP2wrDNbMe3Mjf2ksiK3Hjghg7F2DN9l/omZZl5Mmez2QRk0q4WUUB0+1oh9nDwxGdUXJdXPMRZQs352eGaRPV9s2lcMeZFGWBfKJJiw0YgbCMLBaRmXyy4flx6a667Fch55q05QOq2Jg2ANOyZwplhNsjiohVApo7aa21QnNGW5+4GXv8gxK1beBeHSRrhmLXWVh+0aBhErZ7bx1ejxMOhlR6QU4ycNqGyk8/yNGCWkwY7/RCD7UEQek4QszCgDJAzZtfErA0VqHBy9ugQP9pUfUmgCjVYgWNwHFbhBJyEOgSwBuuwARWZmoI6J9PwLfzEocpRpPrT8DP8wqHG0b4UX+E3DiscvRglXIoi81KKPwioHI5x9EooNKWiy0KOc/T6WF4SssrRuzJ9L2VNRXUhJzj6UKYfS4W/q/5wuh/l4M9R9qsU+y2dpoo2hJzkaEET8r6KRONicnRdK9EbUi6raFVIwNGjsrlbpk6ZPi7TbS3fv3LyNjPiEKzG0aG0tvNb6xw90/whe6ONjnJcUxobHDUqQ8bIOW79BVBLBwhfSmPKdAIAAE4EAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAABkABQBzaW1wbGVtb2RlbC9jb25zdGFudHMucGtsRkIBAFqAAikuUEsHCG0vCVcEAAAABAAAAFBLAwQAAAgIAAAAAAAAAAAAAAAAAAAAAAAAEwA7AHNpbXBsZW1vZGVsL3ZlcnNpb25GQjcAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWjMKUEsHCNGeZ1UCAAAAAgAAAFBLAQIAAAAACAgAAAAAAAAhOZuwWAAAAFgAAAAUAAAAAAAAAAAAAAAAAAAAAABzaW1wbGVtb2RlbC9kYXRhLnBrbFBLAQIAABQACAgIAAAAAABUhNyGggEAACADAAAdAAAAAAAAAAAAAAAAAKgAAABzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weVBLAQIAABQACAgIAAAAAABfSmPKdAIAAE4EAAAnAAAAAAAAAAAAAAAAAJICAABzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weS5kZWJ1Z19wa2xQSwECAAAAAAgIAAAAAAAAbS8JVwQAAAAEAAAAGQAAAAAAAAAAAAAAAACEBQAAc2ltcGxlbW9kZWwvY29uc3RhbnRzLnBrbFBLAQIAAAAACAgAAAAAAADRnmdVAgAAAAIAAAATAAAAAAAAAAAAAAAAANQFAABzaW1wbGVtb2RlbC92ZXJzaW9uUEsGBiwAAAAAAAAAHgMtAAAAAAAAAAAABQAAAAAAAAAFAAAAAAAAAGoBAAAAAAAAUgYAAAAAAABQSwYHAAAAALwHAAAAAAAAAQAAAFBLBQYAAAAABQAFAGoBAABSBgAAAAA=", + "total_parts": 1 + } + - do: + bulk: + index: test-index-with-sparse-vector + refresh: true + body: | + {"index": {}} + {"source_text": "my words comforter", "ml.tokens":{"my":1.0, "words":1.0,"comforter":1.0}} + {"index": {}} + {"source_text": "the machine is leaking", "ml.tokens":{"the":1.0,"machine":1.0,"is":1.0,"leaking":1.0}} + {"index": {}} + {"source_text": "these are my words", "ml.tokens":{"these":1.0,"are":1.0,"my":1.0,"words":1.0}} + {"index": {}} + {"source_text": "the octopus comforter smells", "ml.tokens":{"the":1.0,"octopus":1.0,"comforter":1.0,"smells":1.0}} + {"index": {}} + {"source_text": "the octopus comforter is leaking", "ml.tokens":{"the":1.0,"octopus":1.0,"comforter":1.0,"is":1.0,"leaking":1.0}} + {"index": {}} + {"source_text": "washing machine smells", "ml.tokens":{"washing":1.0,"machine":1.0,"smells":1.0}} + {"index": { "_id": "pinned_doc1" }} + {"source_text": "unrelated pinned doc", "ml.tokens":{"unrelated":1.0,"pinned":1.0,"doc":1.0}} + {"index": { "_id": "pinned_doc2" }} + {"source_text": "another unrelated pinned doc", "ml.tokens":{"another":1.0, "unrelated":1.0,"pinned":1.0,"doc":1.0}} + + - do: + ml.start_trained_model_deployment: + model_id: text_expansion_model + wait_for: started + + - do: + query_ruleset.put: + ruleset_id: combined-ruleset + body: + rules: + - rule_id: rule1 + type: pinned + criteria: + - type: exact + metadata: foo + values: [ bar ] + actions: + ids: + - 'pinned_doc1' + - rule_id: rule2 + type: pinned + criteria: + - type: exact + metadata: foo + values: [ baz ] + actions: + docs: + - '_index': 'test-index-with-sparse-vector' + '_id': 'pinned_doc2' + - do: + search: + body: + query: + rule_query: + organic: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + match_criteria: + foo: bar + ruleset_id: combined-ruleset + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: 'pinned_doc1' } + + - do: + search: + body: + query: + rule_query: + organic: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + match_criteria: + foo: baz + ruleset_id: combined-ruleset + + - match: { hits.total.value: 5 } + - match: { hits.hits.0._id: 'pinned_doc2' } + + - do: + search: + body: + query: + rule_query: + organic: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + match_criteria: + foo: puggle + ruleset_id: combined-ruleset + + - match: { hits.total.value: 4 } + diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java index 3882b6c61bb2c..11d2945a97354 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java @@ -111,18 +111,6 @@ private RuleQueryBuilder( throw new IllegalArgumentException("rulesetId must not be null or empty"); } - // PinnedQueryBuilder will return an error if we attempt to return more than the maximum number of - // pinned hits. Here, we truncate matching rules rather than return an error. - if (pinnedIds != null && pinnedIds.size() > MAX_NUM_PINNED_HITS) { - HeaderWarning.addWarning("Truncating query rule pinned hits to " + MAX_NUM_PINNED_HITS + " documents"); - pinnedIds = pinnedIds.subList(0, MAX_NUM_PINNED_HITS); - } - - if (pinnedDocs != null && pinnedDocs.size() > MAX_NUM_PINNED_HITS) { - HeaderWarning.addWarning("Truncating query rule pinned hits to " + MAX_NUM_PINNED_HITS + " documents"); - pinnedDocs = pinnedDocs.subList(0, MAX_NUM_PINNED_HITS); - } - this.organicQuery = organicQuery; this.matchCriteria = matchCriteria; this.rulesetId = rulesetId; @@ -174,6 +162,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected Query doToQuery(SearchExecutionContext context) throws IOException { + // NOTE: this is old query logic, as in 8.12.2+ and 8.13.0+ we will always rewrite this query + // into a pinned query or the organic query. This logic remains here for backwards compatibility + // with coordinator nodes running versions 8.10.0 - 8.12.1. if ((pinnedIds != null && pinnedIds.isEmpty() == false) && (pinnedDocs != null && pinnedDocs.isEmpty() == false)) { throw new IllegalArgumentException("applied rules contain both pinned ids and pinned docs, only one of ids or docs is allowed"); } @@ -187,21 +178,25 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { } else { return organicQuery.toQuery(context); } - } - @SuppressWarnings("unchecked") @Override - protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { - if (pinnedIds != null || pinnedDocs != null) { - return this; - } else if (pinnedIdsSupplier != null || pinnedDocsSupplier != null) { - List identifiedPinnedIds = pinnedIdsSupplier != null ? pinnedIdsSupplier.get() : null; - List identifiedPinnedDocs = pinnedDocsSupplier != null ? pinnedDocsSupplier.get() : null; - if (identifiedPinnedIds == null && identifiedPinnedDocs == null) { - return this; // not executed yet + protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) { + if (pinnedIdsSupplier != null && pinnedDocsSupplier != null) { + List identifiedPinnedIds = pinnedIdsSupplier.get(); + List identifiedPinnedDocs = pinnedDocsSupplier.get(); + if (identifiedPinnedIds == null || identifiedPinnedDocs == null) { + return this; // Not executed yet + } else if (identifiedPinnedIds.isEmpty() && identifiedPinnedDocs.isEmpty()) { + return organicQuery; // Nothing to pin here + } else if (identifiedPinnedIds.isEmpty() == false && identifiedPinnedDocs.isEmpty() == false) { + throw new IllegalArgumentException( + "applied rules contain both pinned ids and pinned docs, only one of ids or docs is allowed" + ); + } else if (identifiedPinnedIds.isEmpty() == false) { + return new PinnedQueryBuilder(organicQuery, truncateList(identifiedPinnedIds).toArray(new String[0])); } else { - return new RuleQueryBuilder(organicQuery, matchCriteria, rulesetId, identifiedPinnedIds, identifiedPinnedDocs, null, null); + return new PinnedQueryBuilder(organicQuery, truncateList(identifiedPinnedDocs).toArray(new Item[0])); } } @@ -244,18 +239,19 @@ public void onFailure(Exception e) { }); }); - QueryBuilder newOrganicQuery = organicQuery.rewrite(queryRewriteContext); - RuleQueryBuilder rewritten = new RuleQueryBuilder( - newOrganicQuery, - matchCriteria, - this.rulesetId, - null, - null, - pinnedIdsSetOnce::get, - pinnedDocsSetOnce::get - ); - rewritten.boost(this.boost); - return rewritten; + return new RuleQueryBuilder(organicQuery, matchCriteria, this.rulesetId, null, null, pinnedIdsSetOnce::get, pinnedDocsSetOnce::get) + .boost(this.boost) + .queryName(this.queryName); + } + + private List truncateList(List input) { + // PinnedQueryBuilder will return an error if we attempt to return more than the maximum number of + // pinned hits. Here, we truncate matching rules rather than return an error. + if (input.size() > MAX_NUM_PINNED_HITS) { + HeaderWarning.addWarning("Truncating query rule pinned hits to " + MAX_NUM_PINNED_HITS + " documents"); + return input.subList(0, MAX_NUM_PINNED_HITS); + } + return input; } @Override diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilderTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilderTests.java index 1d19011fdd4a9..d4ab8d8f8e6e8 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilderTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilderTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.application.LocalStateEnterpriseSearch; +import org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules; import org.hamcrest.Matchers; import java.io.IOException; @@ -62,7 +63,7 @@ protected void doAssertLuceneQuery(RuleQueryBuilder queryBuilder, Query query, S @Override protected Collection> getPlugins() { - return Collections.singletonList(LocalStateEnterpriseSearch.class); + return List.of(LocalStateEnterpriseSearch.class, SearchBusinessRules.class); } public void testIllegalArguments() { From 2e5875de8b489360ab2b61c4cab7b387988c4178 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 15 Feb 2024 16:32:01 +0100 Subject: [PATCH 67/78] [Connectors Secrets API] Move null checks from constructor to validate --- .../secrets/ConnectorSecretsConstants.java | 14 ++++++++++++++ .../action/DeleteConnectorSecretRequest.java | 8 ++++++-- .../secrets/action/GetConnectorSecretRequest.java | 6 +++++- .../action/DeleteConnectorSecretActionTests.java | 12 +++++++++++- .../action/GetConnectorSecretActionTests.java | 11 ++++++++++- 5 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/ConnectorSecretsConstants.java diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/ConnectorSecretsConstants.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/ConnectorSecretsConstants.java new file mode 100644 index 0000000000000..9c7f192c19248 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/ConnectorSecretsConstants.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.secrets; + +public class ConnectorSecretsConstants { + + public static final String CONNECTOR_SECRET_ID_NULL_OR_EMPTY_MESSAGE = "[id] of the connector secret cannot be null or empty."; + +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretRequest.java index 183362f64ea8f..efc8b2b6c660d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsConstants; import java.io.IOException; import java.util.Objects; @@ -23,7 +24,7 @@ public class DeleteConnectorSecretRequest extends ActionRequest { private final String id; public DeleteConnectorSecretRequest(String id) { - this.id = Objects.requireNonNull(id); + this.id = id; } public DeleteConnectorSecretRequest(StreamInput in) throws IOException { @@ -46,7 +47,10 @@ public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (Strings.isNullOrEmpty(id)) { - validationException = addValidationError("id missing", validationException); + validationException = addValidationError( + ConnectorSecretsConstants.CONNECTOR_SECRET_ID_NULL_OR_EMPTY_MESSAGE, + validationException + ); } return validationException; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretRequest.java index cf1cc0f563eba..e8792dd3755bd 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsConstants; import java.io.IOException; import java.util.Objects; @@ -46,7 +47,10 @@ public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (Strings.isNullOrEmpty(id)) { - validationException = addValidationError("id missing", validationException); + validationException = addValidationError( + ConnectorSecretsConstants.CONNECTOR_SECRET_ID_NULL_OR_EMPTY_MESSAGE, + validationException + ); } return validationException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretActionTests.java index 5d9127527fc3a..503aa0a6304cd 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretActionTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretActionTests.java @@ -9,8 +9,10 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsConstants; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.NULL_STRING; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -29,6 +31,14 @@ public void testValidate_WhenConnectorSecretIdIsEmpty_ExpectValidationError() { ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); assertThat(exception, notNullValue()); - assertThat(exception.getMessage(), containsString("id missing")); + assertThat(exception.getMessage(), containsString(ConnectorSecretsConstants.CONNECTOR_SECRET_ID_NULL_OR_EMPTY_MESSAGE)); + } + + public void testValidate_WhenConnectorSecretIdIsNull_ExpectValidationError() { + DeleteConnectorSecretRequest requestWithMissingConnectorId = new DeleteConnectorSecretRequest(NULL_STRING); + ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSecretsConstants.CONNECTOR_SECRET_ID_NULL_OR_EMPTY_MESSAGE)); } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretActionTests.java index 9fc01e56ee5a0..c60aeffa2911a 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretActionTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretActionTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsConstants; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils; import static org.hamcrest.Matchers.containsString; @@ -29,6 +30,14 @@ public void testValidate_WhenConnectorSecretIdIsEmpty_ExpectValidationError() { ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); assertThat(exception, notNullValue()); - assertThat(exception.getMessage(), containsString("id missing")); + assertThat(exception.getMessage(), containsString(ConnectorSecretsConstants.CONNECTOR_SECRET_ID_NULL_OR_EMPTY_MESSAGE)); + } + + public void testValidate_WhenConnectorSecretIdIsNull_ExpectValidationError() { + GetConnectorSecretRequest requestWithMissingConnectorId = new GetConnectorSecretRequest(""); + ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSecretsConstants.CONNECTOR_SECRET_ID_NULL_OR_EMPTY_MESSAGE)); } } From 2e58cb91a575550d6b8fc4b09a1f3cb60ec81ed2 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 15 Feb 2024 16:39:22 +0100 Subject: [PATCH 68/78] Reduce InternalBinaryRange and InternalRandomSampler in a streaming fashion (#105541) --- .../bucket/range/InternalBinaryRange.java | 60 +++++++------------ .../sampler/random/InternalRandomSampler.java | 16 +++-- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java index f81a5ffbcaf18..2b5bcd9931f6e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java @@ -12,18 +12,18 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorReducer; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; +import org.elasticsearch.search.aggregations.bucket.FixedMultiBucketAggregatorsReducer; import org.elasticsearch.search.aggregations.support.SamplingContext; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -248,50 +248,32 @@ protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceCont return new AggregatorReducer() { - final List aggregations = new ArrayList<>(size); + final FixedMultiBucketAggregatorsReducer reducer = new FixedMultiBucketAggregatorsReducer<>( + reduceContext, + size, + getBuckets() + ) { + + @Override + protected Bucket createBucket(Bucket proto, long docCount, InternalAggregations aggregations) { + return new Bucket(proto.format, proto.keyed, proto.key, proto.from, proto.to, docCount, aggregations); + } + }; @Override public void accept(InternalAggregation aggregation) { - aggregations.add((InternalBinaryRange) aggregation); + InternalBinaryRange binaryRange = (InternalBinaryRange) aggregation; + reducer.accept(binaryRange.getBuckets()); } @Override public InternalAggregation get() { - reduceContext.consumeBucketsAndMaybeBreak(buckets.size()); - long[] docCounts = new long[buckets.size()]; - InternalAggregations[][] aggs = new InternalAggregations[buckets.size()][]; - for (int i = 0; i < aggs.length; ++i) { - aggs[i] = new InternalAggregations[aggregations.size()]; - } - for (int i = 0; i < aggregations.size(); ++i) { - InternalBinaryRange range = aggregations.get(i); - if (range.buckets.size() != buckets.size()) { - throw new IllegalStateException( - "Expected [" + buckets.size() + "] buckets, but got [" + range.buckets.size() + "]" - ); - } - for (int j = 0; j < buckets.size(); ++j) { - Bucket bucket = range.buckets.get(j); - docCounts[j] += bucket.docCount; - aggs[j][i] = bucket.aggregations; - } - } - List buckets = new ArrayList<>(getBuckets().size()); - for (int i = 0; i < getBuckets().size(); ++i) { - Bucket b = getBuckets().get(i); - buckets.add( - new Bucket( - format, - keyed, - b.key, - b.from, - b.to, - docCounts[i], - InternalAggregations.reduce(Arrays.asList(aggs[i]), reduceContext) - ) - ); - } - return new InternalBinaryRange(name, format, keyed, buckets, metadata); + return new InternalBinaryRange(name, format, keyed, reducer.get(), metadata); + } + + @Override + public void close() { + Releasables.close(reducer); } }; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java index c17056cecc053..4dde9cc67b975 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java @@ -10,8 +10,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorReducer; +import org.elasticsearch.search.aggregations.AggregatorsReducer; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; @@ -20,8 +22,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; public class InternalRandomSampler extends InternalSingleBucketAggregation implements Sampler { @@ -79,24 +79,28 @@ protected InternalSingleBucketAggregation newAggregation(String name, long docCo protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceContext, int size) { return new AggregatorReducer() { long docCount = 0L; - final List subAggregationsList = new ArrayList<>(size); + final AggregatorsReducer subAggregatorReducer = new AggregatorsReducer(reduceContext, size); @Override public void accept(InternalAggregation aggregation) { docCount += ((InternalSingleBucketAggregation) aggregation).getDocCount(); - subAggregationsList.add(((InternalSingleBucketAggregation) aggregation).getAggregations()); + subAggregatorReducer.accept(((InternalSingleBucketAggregation) aggregation).getAggregations()); } @Override public InternalAggregation get() { - InternalAggregations aggs = InternalAggregations.reduce(subAggregationsList, reduceContext); + InternalAggregations aggs = subAggregatorReducer.get(); if (reduceContext.isFinalReduce() && aggs != null) { SamplingContext context = buildContext(); aggs = InternalAggregations.from(aggs.asList().stream().map(agg -> agg.finalizeSampling(context)).toList()); } - return newAggregation(getName(), docCount, aggs); } + + @Override + public void close() { + Releasables.close(subAggregatorReducer); + } }; } From a8d2722c5969b19cb442b23fbf4c09436bfc20d5 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 15 Feb 2024 10:39:47 -0500 Subject: [PATCH 69/78] [ESQL] Inverse trig function argument descriptions (#105525) --- .../qa/testFixtures/src/main/resources/show.csv-spec | 10 +++++----- .../esql/expression/function/scalar/math/Acos.java | 5 ++++- .../esql/expression/function/scalar/math/Asin.java | 5 ++++- .../esql/expression/function/scalar/math/Atan.java | 5 ++++- .../esql/expression/function/scalar/math/Atan2.java | 4 ++-- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index 8d86563f78938..6887a1bbe9069 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -6,15 +6,15 @@ v:long ; # TODO: switch this test to ``&format=csv&delimiter=|` output -showFunctions#[skip:-8.12.99] +showFunctions#[skip:-8.13.99] show functions; name:keyword | synopsis:keyword | argNames:keyword | argTypes:keyword | argDescriptions:keyword |returnType:keyword | description:keyword | optionalArgs:boolean | variadic:boolean | isAggregation:boolean abs |"double|integer|long|unsigned_long abs(n:double|integer|long|unsigned_long)" |n |"double|integer|long|unsigned_long" | "" |"double|integer|long|unsigned_long" | "Returns the absolute value." | false | false | false -acos |"double acos(n:double|integer|long|unsigned_long)" |n |"double|integer|long|unsigned_long" | "" |double | "The arccosine of an angle, expressed in radians." | false | false | false -asin |"double asin(n:double|integer|long|unsigned_long)"|n |"double|integer|long|unsigned_long" | "" |double | "Inverse sine trigonometric function." | false | false | false -atan |"double atan(n:double|integer|long|unsigned_long)" |n |"double|integer|long|unsigned_long" | "" |double | "Inverse tangent trigonometric function." | false | false | false -atan2 |"double atan2(y:double|integer|long|unsigned_long, x:double|integer|long|unsigned_long)" |[y, x] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |["", ""] |double | "The angle between the positive x-axis and the ray from the origin to the point (x , y) in the Cartesian plane." | [false, false] | false | false +acos |"double acos(n:double|integer|long|unsigned_long)" |n |"double|integer|long|unsigned_long" | "Number between -1 and 1" |double | "The arccosine of an angle, expressed in radians." | false | false | false +asin |"double asin(n:double|integer|long|unsigned_long)"|n |"double|integer|long|unsigned_long" | "Number between -1 and 1" |double | "Inverse sine trigonometric function." | false | false | false +atan |"double atan(n:double|integer|long|unsigned_long)" |n |"double|integer|long|unsigned_long" | "A number" |double | "Inverse tangent trigonometric function." | false | false | false +atan2 |"double atan2(y:double|integer|long|unsigned_long, x:double|integer|long|unsigned_long)" |[y, x] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |["y coordinate", "x coordinate"] |double | "The angle between the positive x-axis and the ray from the origin to the point (x , y) in the Cartesian plane." | [false, false] | false | false auto_bucket |"double|date auto_bucket(field:integer|long|double|date, buckets:integer, from:integer|long|double|date|string, to:integer|long|double|date|string)" |[field, buckets, from, to] |["integer|long|double|date", "integer", "integer|long|double|date|string", "integer|long|double|date|string"] |["", "", "", ""] | "double|date" | "Creates human-friendly buckets and returns a datetime value for each row that corresponds to the resulting bucket the row falls into." | [false, false, false, false] | false | false avg |"double avg(field:double|integer|long)" |field |"double|integer|long" | "" |double | "The average of a numeric field." | false | false | true case |"boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version case(condition:boolean, rest...:boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version)" |[condition, rest] |["boolean", "boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version"] |["", ""] |"boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version" | "Accepts pairs of conditions and values. The function returns the value that belongs to the first condition that evaluates to true." | [false, false] | true | false diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Acos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Acos.java index 603ef86af6c64..d2e0e8f025665 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Acos.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Acos.java @@ -22,7 +22,10 @@ */ public class Acos extends AbstractTrigonometricFunction { @FunctionInfo(returnType = "double", description = "The arccosine of an angle, expressed in radians.") - public Acos(Source source, @Param(name = "n", type = { "double", "integer", "long", "unsigned_long" }) Expression n) { + public Acos( + Source source, + @Param(name = "n", type = { "double", "integer", "long", "unsigned_long" }, description = "Number between -1 and 1") Expression n + ) { super(source, n); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Asin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Asin.java index f66409921ad2f..38b70cea0350c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Asin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Asin.java @@ -22,7 +22,10 @@ */ public class Asin extends AbstractTrigonometricFunction { @FunctionInfo(returnType = "double", description = "Inverse sine trigonometric function.") - public Asin(Source source, @Param(name = "n", type = { "double", "integer", "long", "unsigned_long" }) Expression n) { + public Asin( + Source source, + @Param(name = "n", type = { "double", "integer", "long", "unsigned_long" }, description = "Number between -1 and 1") Expression n + ) { super(source, n); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan.java index 8f0ad96f96e8c..071379820922a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan.java @@ -22,7 +22,10 @@ */ public class Atan extends AbstractTrigonometricFunction { @FunctionInfo(returnType = "double", description = "Inverse tangent trigonometric function.") - public Atan(Source source, @Param(name = "n", type = { "double", "integer", "long", "unsigned_long" }) Expression n) { + public Atan( + Source source, + @Param(name = "n", type = { "double", "integer", "long", "unsigned_long" }, description = "A number") Expression n + ) { super(source, n); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java index 5b0a795996440..b69a536c2df84 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java @@ -38,8 +38,8 @@ public class Atan2 extends EsqlScalarFunction { ) public Atan2( Source source, - @Param(name = "y", type = { "double", "integer", "long", "unsigned_long" }) Expression y, - @Param(name = "x", type = { "double", "integer", "long", "unsigned_long" }) Expression x + @Param(name = "y", type = { "double", "integer", "long", "unsigned_long" }, description = "y coordinate") Expression y, + @Param(name = "x", type = { "double", "integer", "long", "unsigned_long" }, description = "x coordinate") Expression x ) { super(source, List.of(y, x)); this.y = y; From 15877f559320f7dcc5c655e24375d009643cd0ec Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 15 Feb 2024 10:42:37 -0500 Subject: [PATCH 70/78] Renew samba fixture expired test certificates (#105561) --- .../transport/ssl/certs/simple/samba4.crt | 36 ++++++------- .../security/authc/ldap/support/ADtrust.jks | Bin 1922 -> 1923 bytes .../security/authc/ldap/support/smb_ca.crt | 36 ++++++------- .../security/authc/ldap/support/smb_cert.crt | 36 ++++++------- .../src/main/resources/smb/certs/ca.key | 50 +++++++++--------- .../src/main/resources/smb/certs/ca.pem | 36 ++++++------- .../src/main/resources/smb/certs/cert.pem | 36 ++++++------- .../src/main/resources/smb/certs/key.pem | 50 +++++++++--------- 8 files changed, 140 insertions(+), 140 deletions(-) diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt index a1f78231fa3c4..aba0948092581 100644 --- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt +++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDTzCCAjegAwIBAgIVAKswcWOE2Y8orNolWoM5EJbBVQKAMA0GCSqGSIb3DQEB -CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu -ZXJhdGVkIENBMB4XDTIxMDIxNTExNDE1NVoXDTI0MDIxNTExNDE1NVowETEPMA0G -A1UEAxMGc2FtYmE0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoNwa -9+QyLYF7q8jFOmd/1WNnV50KAnfBXti9WShk1rTso1YK3QflWtqcn0BsJFfUmRJ1 -79tTTtUmzcqcXXR/ygc8UAal/Cl19h1G4k3Mwf8tlUAjRbH0D+mM6HrZ4+QrAqwb -tDezz8TBAEuVJmTcbdrI6iA9y+I4PReqF//EyQwHeUbKvNHBF0vqYFHqErrBaKsC -VAFp9QSKjF9uRLk4e7PLJDZm9BbVTbtQ76o2SHNMni0UcriqYbi6npIWCLs5xFg5 -dSJ3SwrgBbm/Hg994miPt5CN7l69e7h3ul6/Af3UfMTM02YQ9T3mQ4evW667kAoW -Gxu2Z1cXcsq9+Qj/TwIDAQABo3sweTAdBgNVHQ4EFgQUN818rPYPElgO2oFSXM0c -GXvD4cAwHwYDVR0jBBgwFoAUdiZF7wZYdLDjDO6NuDRebJFFFnIwLAYDVR0RBCUw -I4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAkGA1UdEwQCMAAw -DQYJKoZIhvcNAQELBQADggEBAGZQ+NnsLNLbS5dvU2XQDrstcgq7JQcoajH+Us86 -/arjs9C1T7GID1wCXSlyR2uuFTYBH89UpX3oChLohnJByy1Mhpi8l+R2odhosgym -psnt7uejh9DWQrPeiL00ohSjXy3dha5VUESAKoyT/rga1YNl4qeY1J7RPM4NkP7l -nJFzJkTpdcfgcDj42OEOKRjSxaGKTQu5OP6/EXmpxxUdXpfoRfP0IiyCHSNl2gfJ -zxFvVlB2cy8O0EnJ4DdpRpzeTW8pMXGZ7flUSvOhT2rn2K1eqRL9q9LRJn/G/5DP -0u2HxjrZSfn40AS1jcasy6Lcvc7Mwz4W0cfc/tSA67WrFzw= +MIIDTjCCAjagAwIBAgIUJFzoNwmi+322wZTgemjEV35/yrIwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjQwMjE1MTQ0ODM3WhcNMjcwMjE0MTQ0ODM3WjARMQ8wDQYD +VQQDEwZzYW1iYTQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFrD2n +loHuTCCbcX1pqmRnVj7SXcXsBjY9McXxvpEQ4kCdUBlDp3wC9bkGnt6jxlAvEaXn +ZXHg3DqJbD2GU9vRMapcVR4ptAvZmgpEJBU8FA8E1T1EeMhhIoWsYq2crOf/SLQY +g23tO1vmV5+kPD23KiTzj8errYF1Si/6b0IH7XuRUAUJAJYm4gN3YvNqIUYxO/WG +cOlxp/ssIC7Xj6E8Mcdpcfgm+AzLR/x2edl524GAbLqzuKM2bUoMroeNVwVRYPz5 +9iCi+rRxAlI9M2eIm7Gl+qjXUXN1BQAEpkXcxBxDWaIEomd7rR9yudHWvBl8J3+V +jJBgqhecOq/Z+bqbAgMBAAGjezB5MB0GA1UdDgQWBBSbPXW27/QMTZuLD13NUSmQ +0kxaCDAfBgNVHSMEGDAWgBTZMPxziaiEFpaGeNGBxTr+sWGeXTAsBgNVHREEJTAj +hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABgglsb2NhbGhvc3QwCQYDVR0TBAIwADAN +BgkqhkiG9w0BAQsFAAOCAQEAkbnWUXSooivT8FW2PfdTLYDBi18ssdH5g2KU1fNi +E0z+85X1IZaWikGZi3fyj28+GPvRr588EmpPP+IIW9WM5rx3/Gh7elknpwTBYwPo +whuLrKPofEXeM1nUi6nC9QfKDoLWC4B2ztXLD+gcqVHfO/3Q9cs88z9IK4fMAlHu +2dof6SlqVI7AAsqCHaIIuiABW3U81URdtwCJqKt2SUQmpRh6PyAT8rqP7f//gBFF +KyLGOhlEikm+DpFvLNtBHFLL/u9P6u9zV5xyqI4K3wGMAvrTbpicXLJQC6bdIEJR +gFSUL8f6cal5l0ncN6dT3RnpAWwt88xzg8OSMDPWYAyHbw== -----END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ADtrust.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ADtrust.jks index ab828467407526b0e43fb2e265643beafde020e9..b497bdbd0585c87cd8903bbbc722da79429fd63d 100644 GIT binary patch delta 717 zcmV;;0y6!A4}%Yo8#IlrK3Mty1z0XMFgXAK15Pl414=N00y3Zj0s#U76#(mH6@=ac zg)S`xbvsy*CupQ#IkuX|syJ}S~pB8NFO*2R-{YWi=dXpE&#>kDxvyZ4!G07ywwXz#LuT%KUc1Zw z&4kU~^(K>vcjgKAp!F8ow1;7gR68;MNJ5i+bOBtY7H*l7GMg2@$66LGZ1^?ytfkP`=YmJt z?|4@zNF`9e)INC!G3R`Xh2?cl^~u_bs%)u2SbF5VtUaQJgre$!SA!d#dVdB>YR>`# z0RRD`Q!r659R>qc9S#H*1QgjY{Bwz@gcg>Dc+r8yI{vX?o?S2>1_MP3&x+X-ypPMfE(BeEKZ`m^Y$+)XmexE z-?1>)H{97ulw#Lra9i;!Md1>h8w7k#P?VT4V}4M&8iJjirLgS5nO%#s26qsy*|NXQ zh>@~&%*KW)u$F_UzXzb@d;?vTiK-O|23~5d4rhk zZo4`EI9f!Ggc^gYql`{BpS5}dJfX9A1NH+Ht^Cb|iBQg;AC4iFB8Kr&z9B+@@OMf4 delta 716 zcmZqXZ{nXIZCt)|e!d|CYlNPufh7Y2v#&uDv!_85li>npCPpSEkyOj}hVv1c^=dn= zoxjnhv_D(E>dZucFKI&qBSTX|Lt_gAvnX+1BNHU<#5nVbuNCUg7O%)w%VuAfJl$s* zzl!e7w@eKeu4_&BIsJE{_ScI-dnErhE3b{2rWkwEeEFB<U*x- zCwP3}yw*C_4_3U5ZPR8RP>E)L@~qY2saDG5SG=!F<5pCi3hOgqX+Ci(tNAr^7Jq>b z>r#vA1CKluUc}vuI2&_Fquz?YV@L8+v&n^9wX|PLY9A@ymUH#en?63>#IL(^R3_}+ z)BQb5ZduRyZ#%@r=L_wX-_aeh?#wMRa{`Vx2Pf$ zGb01z;$VY716elaP+2|}F&2?BHP`oS5hWWQ^StZbVG@@!(N(O-KprHm%pzeR)__&P zXx z+LG~9Lzv)-S!vZ<9(W!-H1AcDw1X7)M57mP!`d<~PTy8PJ=dpHiQ90S#n}t7$GT31 zNd-Nut-kCY9B5nge7W|w%*-!U1#?wha`volFE}0UwInri?kk-Ocj{g~RC@Z`*r@u6 z&06Ez2OQknj6&JxM%wk3T->pgMOs_yhSdS~19PH}{#+|~XpTpgY3VH4Wmg|u43bv( w82O>yHemA(clGYG|C#zVH&-%$V-{KXc=w{7dBSgQ{0y~ZUHbK4&v&;#07ip77ytkO diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt index df7b319d1e313..a3281c92af704 100644 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt +++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDSTCCAjGgAwIBAgIUZTmHgM9YKX8muNbP2IYiv2sfeswwDQYJKoZIhvcNAQEL -BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l -cmF0ZWQgQ0EwHhcNMjEwMjE1MTEzODA2WhcNMjQwMjE1MTEzODA2WjA0MTIwMAYD -VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1zqGsmaweuY5dMpg8kLdnt -AoDQ1yqQ+Zf7YSv10RK8Gf2DI61cliFd2Ten9Kd3RBg5RKtEXxHDiQkvSam+Eceh -noV+BfA6DYGGlpnAJFsH5OaFQOUqZJPqDet1Xqh6ylaOMASDyMpqg+sDag9wLAWl -OHvA4kgg6F7ZWM1cwig6D4i4Y+U2k3G1KivrGSvEc7Zs1dLsjg4tYfW7bCSQu7yL -92oepozP9rgXF58SvR+4i1iu7P1r769R1WUMCHJ98f9rXDHHdsS1WMfTsuTP9h2t -TJSBGkrQKtZ0v+7oe2yKArFGF+U9FuZVD1D2Fc4dTDMxpCALF6TeIK6FFxC7FSEC -AwEAAaNTMFEwHQYDVR0OBBYEFHYmRe8GWHSw4wzujbg0XmyRRRZyMB8GA1UdIwQY -MBaAFHYmRe8GWHSw4wzujbg0XmyRRRZyMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI -hvcNAQELBQADggEBADZ5ddmAODL7QeZHkUrUKtGkMWK0WpdlQKx3XkxDod4DM8cI -OaKVUhJznt5ShPCMYZ3+EbTAZhaJUnxdUrulTh8UW0ezDFcnninkzVJ1K2tlYurF -CgrOpr+ni1T08pWrOQHKgFYRqJpme7TgScXCnuqCG0AaC5Ey6O1WhmjRl7aXbUx1 -IgsxtjjN0F3GispWGlLhfXvTR1NRPXLnpyv2aWn0enCdJURsvKuHcMtXSqRlWZ3q -LNDcfunhIuX7MzJ75DytM9vAQEeGMlUHnVk+jXTRuKUEGysq2DrAB8CcW8X5rRHC -nEhqNXWaHabV4NFSGyDxWfCHPVCzuEcni83/jyk= +MIIDSjCCAjKgAwIBAgIVAOtlFYTeA4VeumO8mgMu/nMuJRxRMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTI0MDIxNTE0NDc0NFoXDTI3MDIxNDE0NDc0NFowNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDXFQRa1KKGfLNaRbiwHeK2 +kTBab7tqVS6fFmzsTTNIKf1ILYF6kxfHxsiN1jM8tVU5n0Kirqe8eRVAQn/4Hj09 +I8dhVLmvYI/BfioZWU0MXLk0aA2nUm5dk9eVkLaZSaVDZu57XrvL/s2Ezd71JpOJ +d+l9zQLNxrhvhTgw3sV53KURQip8UoisSXjbyBG5eq3Ck1DfdZ0lrXoTM/cgGp3i +yMk/oPUW2rSHYYxUOzH/SEKTfXQBXKUWbpmTMpsVv8daFi1s+DX2rKXQ1+eCR9rv +eFcoSCVQv9Q+eQcx53yLheV1TvXJ2oqqbKlBWHrkvKw9ooaEouqBV4Mbnnp/BmrP +AgMBAAGjUzBRMB0GA1UdDgQWBBTZMPxziaiEFpaGeNGBxTr+sWGeXTAfBgNVHSME +GDAWgBTZMPxziaiEFpaGeNGBxTr+sWGeXTAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQARiXwCC8afr98gtGGAG9gJLE6ivfP2Lypoc2PO37Ew1zfc +2UqUYtdmcFvxK0XhEpwbBHxOUJSYMWN+ULoagp2cpbDswZldi7MGdxCu2bK/zYiR +snXMxoYqsJaDqL8HoOV8A12ViaoVCQZeaq0OZ4/NrTRCcPzx+Yru9ZWfgG+4gDbC +hxnDJBY81ju8fhfAcCGuxApPE9fkwFwwVr5nwCbAl7Qotliol/h1eV35hPocyUVz +AxOW2Foc7ehru6NynnEAqOhjcTwMlZCKt0b4kPz2kd+9vL1ZKc/hjNX6Q6sW1N6V +z9Gu6uSsIiM8brs5/zhaRI6EGoOqo4xON5+1ejyh -----END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt index a1f78231fa3c4..aba0948092581 100644 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt +++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDTzCCAjegAwIBAgIVAKswcWOE2Y8orNolWoM5EJbBVQKAMA0GCSqGSIb3DQEB -CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu -ZXJhdGVkIENBMB4XDTIxMDIxNTExNDE1NVoXDTI0MDIxNTExNDE1NVowETEPMA0G -A1UEAxMGc2FtYmE0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoNwa -9+QyLYF7q8jFOmd/1WNnV50KAnfBXti9WShk1rTso1YK3QflWtqcn0BsJFfUmRJ1 -79tTTtUmzcqcXXR/ygc8UAal/Cl19h1G4k3Mwf8tlUAjRbH0D+mM6HrZ4+QrAqwb -tDezz8TBAEuVJmTcbdrI6iA9y+I4PReqF//EyQwHeUbKvNHBF0vqYFHqErrBaKsC -VAFp9QSKjF9uRLk4e7PLJDZm9BbVTbtQ76o2SHNMni0UcriqYbi6npIWCLs5xFg5 -dSJ3SwrgBbm/Hg994miPt5CN7l69e7h3ul6/Af3UfMTM02YQ9T3mQ4evW667kAoW -Gxu2Z1cXcsq9+Qj/TwIDAQABo3sweTAdBgNVHQ4EFgQUN818rPYPElgO2oFSXM0c -GXvD4cAwHwYDVR0jBBgwFoAUdiZF7wZYdLDjDO6NuDRebJFFFnIwLAYDVR0RBCUw -I4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAkGA1UdEwQCMAAw -DQYJKoZIhvcNAQELBQADggEBAGZQ+NnsLNLbS5dvU2XQDrstcgq7JQcoajH+Us86 -/arjs9C1T7GID1wCXSlyR2uuFTYBH89UpX3oChLohnJByy1Mhpi8l+R2odhosgym -psnt7uejh9DWQrPeiL00ohSjXy3dha5VUESAKoyT/rga1YNl4qeY1J7RPM4NkP7l -nJFzJkTpdcfgcDj42OEOKRjSxaGKTQu5OP6/EXmpxxUdXpfoRfP0IiyCHSNl2gfJ -zxFvVlB2cy8O0EnJ4DdpRpzeTW8pMXGZ7flUSvOhT2rn2K1eqRL9q9LRJn/G/5DP -0u2HxjrZSfn40AS1jcasy6Lcvc7Mwz4W0cfc/tSA67WrFzw= +MIIDTjCCAjagAwIBAgIUJFzoNwmi+322wZTgemjEV35/yrIwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjQwMjE1MTQ0ODM3WhcNMjcwMjE0MTQ0ODM3WjARMQ8wDQYD +VQQDEwZzYW1iYTQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFrD2n +loHuTCCbcX1pqmRnVj7SXcXsBjY9McXxvpEQ4kCdUBlDp3wC9bkGnt6jxlAvEaXn +ZXHg3DqJbD2GU9vRMapcVR4ptAvZmgpEJBU8FA8E1T1EeMhhIoWsYq2crOf/SLQY +g23tO1vmV5+kPD23KiTzj8errYF1Si/6b0IH7XuRUAUJAJYm4gN3YvNqIUYxO/WG +cOlxp/ssIC7Xj6E8Mcdpcfgm+AzLR/x2edl524GAbLqzuKM2bUoMroeNVwVRYPz5 +9iCi+rRxAlI9M2eIm7Gl+qjXUXN1BQAEpkXcxBxDWaIEomd7rR9yudHWvBl8J3+V +jJBgqhecOq/Z+bqbAgMBAAGjezB5MB0GA1UdDgQWBBSbPXW27/QMTZuLD13NUSmQ +0kxaCDAfBgNVHSMEGDAWgBTZMPxziaiEFpaGeNGBxTr+sWGeXTAsBgNVHREEJTAj +hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABgglsb2NhbGhvc3QwCQYDVR0TBAIwADAN +BgkqhkiG9w0BAQsFAAOCAQEAkbnWUXSooivT8FW2PfdTLYDBi18ssdH5g2KU1fNi +E0z+85X1IZaWikGZi3fyj28+GPvRr588EmpPP+IIW9WM5rx3/Gh7elknpwTBYwPo +whuLrKPofEXeM1nUi6nC9QfKDoLWC4B2ztXLD+gcqVHfO/3Q9cs88z9IK4fMAlHu +2dof6SlqVI7AAsqCHaIIuiABW3U81URdtwCJqKt2SUQmpRh6PyAT8rqP7f//gBFF +KyLGOhlEikm+DpFvLNtBHFLL/u9P6u9zV5xyqI4K3wGMAvrTbpicXLJQC6bdIEJR +gFSUL8f6cal5l0ncN6dT3RnpAWwt88xzg8OSMDPWYAyHbw== -----END CERTIFICATE----- diff --git a/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.key b/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.key index c490e0477f8f5..1f359f3bd9c71 100644 --- a/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.key +++ b/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.key @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzXOoayZrB65jl0ymDyQt2e0CgNDXKpD5l/thK/XRErwZ/YMj -rVyWIV3ZN6f0p3dEGDlEq0RfEcOJCS9Jqb4Rx6GehX4F8DoNgYaWmcAkWwfk5oVA -5Spkk+oN63VeqHrKVo4wBIPIymqD6wNqD3AsBaU4e8DiSCDoXtlYzVzCKDoPiLhj -5TaTcbUqK+sZK8RztmzV0uyODi1h9btsJJC7vIv3ah6mjM/2uBcXnxK9H7iLWK7s -/Wvvr1HVZQwIcn3x/2tcMcd2xLVYx9Oy5M/2Ha1MlIEaStAq1nS/7uh7bIoCsUYX -5T0W5lUPUPYVzh1MMzGkIAsXpN4groUXELsVIQIDAQABAoIBAQC95K7ACsxWVysZ -xvCdghUXHed4ZI7bexAXF3OjDBtoM4/aL4GkVLU2teV1ebm5p5S6xwPfQNyWMKLS -aHCvgUwxtgIs5GRcu2uMUneUOHHh6ZP6NSPcLKi1xlmDTgJlusiV4+oh4iSOYpD3 -gTpgdo9Z0HI6f/cmL7RXJpDbj2atggKZVeSln0BYYRkNcuskPfCK3mPzOYJF5V9T -YbtnaTij72d9v7w181JRHRJQzL6jN0FiIl8BoUmepws+FMXW3lyX/jZ7unf8idTm -9d1cT7OfKThMzY+OzapJHZLl85rCAq4B617ovDHVx1lbNviBiBoTNQIjSNmgkgXk -WUZ2m0KBAoGBAP1WBkAjNuF1/HdGAgge8bPWU6SqJe1Gy25ZkRLkKxp6701EvC+p -a6OuuFonDZLTuTPDgL/v78HQMwQR+QULerAyfwCJDAXIHMFNSvAP/jzEbg4sKRp/ -0ZVUvjedxtADeHIs3SbNEt1P4A+g5U1Is5XGdeKe16zaxfXfy0VA8onZAoGBAM+c -uuNX9FxRrVDUiEIJJFADD/VePXR7YSbjUoLvLtlms4ObOUrnJZJ0IsWxkY4AnDv/ -F0PVA0uMZtVK28BZt1SsYLrKu4KbGPqJy9ArKCAeWuk3DK19XJ8heUkrprEKmQPq -kbIy+LU9MaUKg5pThh1T+uG3QA7liFixxW4KB9CJAoGBAPCWZMM+bh06LrSbMMzD -jmlqzu6fg/tN815uAx98vw6b1217LHjbHdVJ1dwQIIzjM4xcS4Z8eCaI8hoYc3R+ -DVsN6Zz5ighnnh9ZpyRLG/hb1+TvvW1kHAcEfs1Usn3T/ev4fWIe+Z5h//j3pSx7 -Mcm4uzWoAk0vSzcQ/PtdDbkpAoGAf9k+ZV32hxttJYeb2T6T9AnOvAUqxx5rd20p -lKQCL7LE/ViLcYriYkvOVfvBbLcHfxLZmtYET3PIp7SPmuYktanpb14FFqq4OSC+ -OBU7gnvu9AsIbZXzgbM1Y0/UONYT4IuE6T3mVoW2mrHc1R52Sn060+DrO8Exs5zV -vavDoDECgYBN2KF83fdixXCIDizrlJb3mbkhvLoOX+Q3iuJcOLvVrnFo9mGuXhXi -+2K9kVxihqgupRBoR8H60iTB/N3KGvqs4nffDu/lTO4nM9bA13CAVB1JzJG2uh1Y -jFc9zdB4obKShS41VjdTtCh5xi1icX+n8/CGb5TCF8W4ORO8mn9TNg== +MIIEoQIBAAKCAQEA1xUEWtSihnyzWkW4sB3itpEwWm+7alUunxZs7E0zSCn9SC2B +epMXx8bIjdYzPLVVOZ9Coq6nvHkVQEJ/+B49PSPHYVS5r2CPwX4qGVlNDFy5NGgN +p1JuXZPXlZC2mUmlQ2bue167y/7NhM3e9SaTiXfpfc0Czca4b4U4MN7FedylEUIq +fFKIrEl428gRuXqtwpNQ33WdJa16EzP3IBqd4sjJP6D1Ftq0h2GMVDsx/0hCk310 +AVylFm6ZkzKbFb/HWhYtbPg19qyl0Nfngkfa73hXKEglUL/UPnkHMed8i4XldU71 +ydqKqmypQVh65LysPaKGhKLqgVeDG556fwZqzwIDAQABAoIBABE76o4Up7Pls2wH +q4Ar2jS4IYsS6jjfpngHkKbGm4CsIf1x7VloW+lyxjMx910P8p+cC/eDzdujoCOh +WyZYgJ5biuhY+kqmjsj5q7SS8UI3qSdyhwA3R1ym3LQWqmr8pF9oP10R/t5JBn1z +uZUkiCyQYoc6sX987YkSFankz9IEdGRg2xJx9i2Iao5oDHmLUqDMzBS2TcKTv7LN +/pZQEPo/Dsvkvo5DDD6a6aBlLBCs5YcoBbtA2HeknSJRPFRay196sIh8yRcQ0WuK +zLeJvh57OK/J/Jm8jUrpYcDKEFDocFHeopRcXCbQlV1bXnuEJDNfAT/KLo5fkT3N +Xa2nrFECgYEA+ipKdR2vJjNXLSJmLO7tl63W+yESg1bLklXqMKf7WKvBEbrpeJgi +3o+HhxBrKKqBz/Y4O9x+xHhYlbWtm/6Wc9K72wmWEMf0hGevjRxM3eqqWB+7qSJ9 +qYXtwANKR/SMongos0Gavts7XLZK6exkE7RypMrU28P1jb+BQsg56n0CgYEA3Bk/ +sIdf+4Fus0KyUemZIp6xPfpUjOlccrasWA2GGfmi8xOTQn++s3ht0V04kMPHm5gH +AQ9S48+pCczDn9JFtjeH0h6dYVi+7of9NqHa3OXt++WL2T2jvum/iaK2PcW0NHqd +3owSJD+rrx4cXImW2LbKUo73L3CcxOapxYaA4DsCgYBPOaN48ZyteWbrWVCIfGZs +Odayk2e8hnlT77eKDzjvfP1Y8xvLYErytvvRz2ZQa6dOyAhJFOxkpkRPrUi89WSK +a7uog6Gt0NVkNT4Ib2T8hrvJysrwpoarcEm6HJCitxTuwyUImAc82Es1clnJOV78 +SpJgFAhTTPzwFi0GjEijNQKBgQC8xKrrLDAV9RyMgleOCVtdZd192oVJlZvEhwep +PXAWNxSahd922Tklk9QcDGfHQSKhP/JB5nKhEClaTlQ5bo57iYTjoX45T3PyAJAb +mxWq/0jtEiKvXz5hLvkngnXq5PV5TPC5PkkQ7crBloGcnCTUGXHM/PDjryHFfk99 +Ka6+oQJ/ZUPIxj8Lvu7oTVEFPczkM0iTgO1YolY/Z/E+a5suS4FaT5U5Ar9W8E5m +7FLh65QXVMsjxnP+DNSqqNuSpqsyTWFMGtwFIueGRbCnCMN+GBy07fMKC7JLnXmB +Vxf3WwwQ5Crncqn87o0m/mhf/ovsXbGFqqRSJrA0vJifi5D36Q== -----END RSA PRIVATE KEY----- diff --git a/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.pem b/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.pem index df7b319d1e313..a3281c92af704 100644 --- a/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.pem +++ b/x-pack/test/smb-fixture/src/main/resources/smb/certs/ca.pem @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDSTCCAjGgAwIBAgIUZTmHgM9YKX8muNbP2IYiv2sfeswwDQYJKoZIhvcNAQEL -BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l -cmF0ZWQgQ0EwHhcNMjEwMjE1MTEzODA2WhcNMjQwMjE1MTEzODA2WjA0MTIwMAYD -VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1zqGsmaweuY5dMpg8kLdnt -AoDQ1yqQ+Zf7YSv10RK8Gf2DI61cliFd2Ten9Kd3RBg5RKtEXxHDiQkvSam+Eceh -noV+BfA6DYGGlpnAJFsH5OaFQOUqZJPqDet1Xqh6ylaOMASDyMpqg+sDag9wLAWl -OHvA4kgg6F7ZWM1cwig6D4i4Y+U2k3G1KivrGSvEc7Zs1dLsjg4tYfW7bCSQu7yL -92oepozP9rgXF58SvR+4i1iu7P1r769R1WUMCHJ98f9rXDHHdsS1WMfTsuTP9h2t -TJSBGkrQKtZ0v+7oe2yKArFGF+U9FuZVD1D2Fc4dTDMxpCALF6TeIK6FFxC7FSEC -AwEAAaNTMFEwHQYDVR0OBBYEFHYmRe8GWHSw4wzujbg0XmyRRRZyMB8GA1UdIwQY -MBaAFHYmRe8GWHSw4wzujbg0XmyRRRZyMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI -hvcNAQELBQADggEBADZ5ddmAODL7QeZHkUrUKtGkMWK0WpdlQKx3XkxDod4DM8cI -OaKVUhJznt5ShPCMYZ3+EbTAZhaJUnxdUrulTh8UW0ezDFcnninkzVJ1K2tlYurF -CgrOpr+ni1T08pWrOQHKgFYRqJpme7TgScXCnuqCG0AaC5Ey6O1WhmjRl7aXbUx1 -IgsxtjjN0F3GispWGlLhfXvTR1NRPXLnpyv2aWn0enCdJURsvKuHcMtXSqRlWZ3q -LNDcfunhIuX7MzJ75DytM9vAQEeGMlUHnVk+jXTRuKUEGysq2DrAB8CcW8X5rRHC -nEhqNXWaHabV4NFSGyDxWfCHPVCzuEcni83/jyk= +MIIDSjCCAjKgAwIBAgIVAOtlFYTeA4VeumO8mgMu/nMuJRxRMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTI0MDIxNTE0NDc0NFoXDTI3MDIxNDE0NDc0NFowNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDXFQRa1KKGfLNaRbiwHeK2 +kTBab7tqVS6fFmzsTTNIKf1ILYF6kxfHxsiN1jM8tVU5n0Kirqe8eRVAQn/4Hj09 +I8dhVLmvYI/BfioZWU0MXLk0aA2nUm5dk9eVkLaZSaVDZu57XrvL/s2Ezd71JpOJ +d+l9zQLNxrhvhTgw3sV53KURQip8UoisSXjbyBG5eq3Ck1DfdZ0lrXoTM/cgGp3i +yMk/oPUW2rSHYYxUOzH/SEKTfXQBXKUWbpmTMpsVv8daFi1s+DX2rKXQ1+eCR9rv +eFcoSCVQv9Q+eQcx53yLheV1TvXJ2oqqbKlBWHrkvKw9ooaEouqBV4Mbnnp/BmrP +AgMBAAGjUzBRMB0GA1UdDgQWBBTZMPxziaiEFpaGeNGBxTr+sWGeXTAfBgNVHSME +GDAWgBTZMPxziaiEFpaGeNGBxTr+sWGeXTAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQARiXwCC8afr98gtGGAG9gJLE6ivfP2Lypoc2PO37Ew1zfc +2UqUYtdmcFvxK0XhEpwbBHxOUJSYMWN+ULoagp2cpbDswZldi7MGdxCu2bK/zYiR +snXMxoYqsJaDqL8HoOV8A12ViaoVCQZeaq0OZ4/NrTRCcPzx+Yru9ZWfgG+4gDbC +hxnDJBY81ju8fhfAcCGuxApPE9fkwFwwVr5nwCbAl7Qotliol/h1eV35hPocyUVz +AxOW2Foc7ehru6NynnEAqOhjcTwMlZCKt0b4kPz2kd+9vL1ZKc/hjNX6Q6sW1N6V +z9Gu6uSsIiM8brs5/zhaRI6EGoOqo4xON5+1ejyh -----END CERTIFICATE----- diff --git a/x-pack/test/smb-fixture/src/main/resources/smb/certs/cert.pem b/x-pack/test/smb-fixture/src/main/resources/smb/certs/cert.pem index a1f78231fa3c4..aba0948092581 100644 --- a/x-pack/test/smb-fixture/src/main/resources/smb/certs/cert.pem +++ b/x-pack/test/smb-fixture/src/main/resources/smb/certs/cert.pem @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDTzCCAjegAwIBAgIVAKswcWOE2Y8orNolWoM5EJbBVQKAMA0GCSqGSIb3DQEB -CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu -ZXJhdGVkIENBMB4XDTIxMDIxNTExNDE1NVoXDTI0MDIxNTExNDE1NVowETEPMA0G -A1UEAxMGc2FtYmE0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoNwa -9+QyLYF7q8jFOmd/1WNnV50KAnfBXti9WShk1rTso1YK3QflWtqcn0BsJFfUmRJ1 -79tTTtUmzcqcXXR/ygc8UAal/Cl19h1G4k3Mwf8tlUAjRbH0D+mM6HrZ4+QrAqwb -tDezz8TBAEuVJmTcbdrI6iA9y+I4PReqF//EyQwHeUbKvNHBF0vqYFHqErrBaKsC -VAFp9QSKjF9uRLk4e7PLJDZm9BbVTbtQ76o2SHNMni0UcriqYbi6npIWCLs5xFg5 -dSJ3SwrgBbm/Hg994miPt5CN7l69e7h3ul6/Af3UfMTM02YQ9T3mQ4evW667kAoW -Gxu2Z1cXcsq9+Qj/TwIDAQABo3sweTAdBgNVHQ4EFgQUN818rPYPElgO2oFSXM0c -GXvD4cAwHwYDVR0jBBgwFoAUdiZF7wZYdLDjDO6NuDRebJFFFnIwLAYDVR0RBCUw -I4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAkGA1UdEwQCMAAw -DQYJKoZIhvcNAQELBQADggEBAGZQ+NnsLNLbS5dvU2XQDrstcgq7JQcoajH+Us86 -/arjs9C1T7GID1wCXSlyR2uuFTYBH89UpX3oChLohnJByy1Mhpi8l+R2odhosgym -psnt7uejh9DWQrPeiL00ohSjXy3dha5VUESAKoyT/rga1YNl4qeY1J7RPM4NkP7l -nJFzJkTpdcfgcDj42OEOKRjSxaGKTQu5OP6/EXmpxxUdXpfoRfP0IiyCHSNl2gfJ -zxFvVlB2cy8O0EnJ4DdpRpzeTW8pMXGZ7flUSvOhT2rn2K1eqRL9q9LRJn/G/5DP -0u2HxjrZSfn40AS1jcasy6Lcvc7Mwz4W0cfc/tSA67WrFzw= +MIIDTjCCAjagAwIBAgIUJFzoNwmi+322wZTgemjEV35/yrIwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjQwMjE1MTQ0ODM3WhcNMjcwMjE0MTQ0ODM3WjARMQ8wDQYD +VQQDEwZzYW1iYTQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFrD2n +loHuTCCbcX1pqmRnVj7SXcXsBjY9McXxvpEQ4kCdUBlDp3wC9bkGnt6jxlAvEaXn +ZXHg3DqJbD2GU9vRMapcVR4ptAvZmgpEJBU8FA8E1T1EeMhhIoWsYq2crOf/SLQY +g23tO1vmV5+kPD23KiTzj8errYF1Si/6b0IH7XuRUAUJAJYm4gN3YvNqIUYxO/WG +cOlxp/ssIC7Xj6E8Mcdpcfgm+AzLR/x2edl524GAbLqzuKM2bUoMroeNVwVRYPz5 +9iCi+rRxAlI9M2eIm7Gl+qjXUXN1BQAEpkXcxBxDWaIEomd7rR9yudHWvBl8J3+V +jJBgqhecOq/Z+bqbAgMBAAGjezB5MB0GA1UdDgQWBBSbPXW27/QMTZuLD13NUSmQ +0kxaCDAfBgNVHSMEGDAWgBTZMPxziaiEFpaGeNGBxTr+sWGeXTAsBgNVHREEJTAj +hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABgglsb2NhbGhvc3QwCQYDVR0TBAIwADAN +BgkqhkiG9w0BAQsFAAOCAQEAkbnWUXSooivT8FW2PfdTLYDBi18ssdH5g2KU1fNi +E0z+85X1IZaWikGZi3fyj28+GPvRr588EmpPP+IIW9WM5rx3/Gh7elknpwTBYwPo +whuLrKPofEXeM1nUi6nC9QfKDoLWC4B2ztXLD+gcqVHfO/3Q9cs88z9IK4fMAlHu +2dof6SlqVI7AAsqCHaIIuiABW3U81URdtwCJqKt2SUQmpRh6PyAT8rqP7f//gBFF +KyLGOhlEikm+DpFvLNtBHFLL/u9P6u9zV5xyqI4K3wGMAvrTbpicXLJQC6bdIEJR +gFSUL8f6cal5l0ncN6dT3RnpAWwt88xzg8OSMDPWYAyHbw== -----END CERTIFICATE----- diff --git a/x-pack/test/smb-fixture/src/main/resources/smb/certs/key.pem b/x-pack/test/smb-fixture/src/main/resources/smb/certs/key.pem index 2def63ef63914..fbdbb5a48e1c2 100644 --- a/x-pack/test/smb-fixture/src/main/resources/smb/certs/key.pem +++ b/x-pack/test/smb-fixture/src/main/resources/smb/certs/key.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAoNwa9+QyLYF7q8jFOmd/1WNnV50KAnfBXti9WShk1rTso1YK -3QflWtqcn0BsJFfUmRJ179tTTtUmzcqcXXR/ygc8UAal/Cl19h1G4k3Mwf8tlUAj -RbH0D+mM6HrZ4+QrAqwbtDezz8TBAEuVJmTcbdrI6iA9y+I4PReqF//EyQwHeUbK -vNHBF0vqYFHqErrBaKsCVAFp9QSKjF9uRLk4e7PLJDZm9BbVTbtQ76o2SHNMni0U -criqYbi6npIWCLs5xFg5dSJ3SwrgBbm/Hg994miPt5CN7l69e7h3ul6/Af3UfMTM -02YQ9T3mQ4evW667kAoWGxu2Z1cXcsq9+Qj/TwIDAQABAoIBAFyLRNio6FVV0Rib -gRyAvwK9FY9KQZ/51b4DY6wPmTQNErdFoSpYiJMkgkb8gTrHbQBDpqY/wEXyS0jJ -7/u0MnDAiOphaM0R7VOStF0t9UcGz+q145UNNCSTcQWu5/w8IKKA8c9U3JYD1CWf -Vkeob0ikPkK0GdlaZJCBNNJpy4T+BOR5D3U7o0Lz3aDExR4ExHWI1+U6xwnOFYxC -VtdKFyzYuT2bOk5E2Y0oi9EhEmJBlIvQ6p3rDPppXmbuTcCNszKal1EyYZePeke/ -iovJZ7+pQS1QsQoXK5el5N2UFM9oAwfARnM8N+yyW5lEV629wVbrYp1WYYefs12e -zGvDqEECgYEA1Vqszj8+RP7FdpCY1jFrvCIRSuIH6KKTD+6mEVcm2GXIZZlI5UGB -U6ZQ+VNxTL6wKPOqqK39jp46HPyvjIwLLwTHNSSLpgAAhBlgR6ZYTW1ImscIsJ5q -dageElmZgRLAmEEF4kx2iJNtTO/YGBqvpf6Q4sbWufO1ARbPQsg7RGMCgYEAwQNL -JPgKtYTzFYnamyq7zWvhfOdWXBVNCT0l0I9FCYKcNtc2yDEme2TOtwkKxcY+1EVj -CTCDabob3EJA6OCAG7GDXrpxbY1o3BB1XtbMnu7KPM/RlcG/qY1qE6kPpCMUevjb -qZfrRkVkH8sTsTQj+Ok4ECBVuUYpJ9BDHNK5fyUCgYAtq8xkFhuxT0xb1hYxe8DR -NAW5nusMfIi4l2CLQ7m4Bwm/3fFByiTyEB8zUA5n3EX/bjGxDBXECtDr1ZeKoYvf -U8mE8b7HGScDIB+BFvW+FU++ei69CBxH9WYCjZWTkL0Tmo+04qNZFx4Foy4B8ux4 -vyaqtN/QTIAJrKVPaWduewKBgC8goL58GhFMTxZZPJlai9SSnNIkoj+Fq/OvjIYq -FU9HJuF1Fxk2dxD2AktK1+iGiVzHPHFH+S5dlOPpAXRbLKyWYV9F4uA/APWKxz3K -8Nd1sse6bpBEaIn7z4TRaNJJBn0oOmpkf7v+wX3J1hsUghwKxfeaDZRZfz8LaPem -tEhJAoGAYikg/Vvw6VUtnDfl0bmC9GRUYrrXpdiz7FlbK6VPe9rs42a5HyrQqVBC -P24wBGfzwFqaTzD/ngodNfnyjNkywChNC18tV3IVbEW5udhX7IJV2Lz52aZBGryK -qt/MXIOtlgjx1hMqfsM3QoMxxyAvwY3tC8vevm3k8yzGfe7Vxb8= +MIIEpAIBAAKCAQEAxaw9p5aB7kwgm3F9aapkZ1Y+0l3F7AY2PTHF8b6REOJAnVAZ +Q6d8AvW5Bp7eo8ZQLxGl52Vx4Nw6iWw9hlPb0TGqXFUeKbQL2ZoKRCQVPBQPBNU9 +RHjIYSKFrGKtnKzn/0i0GINt7Ttb5lefpDw9tyok84/Hq62BdUov+m9CB+17kVAF +CQCWJuIDd2LzaiFGMTv1hnDpcaf7LCAu14+hPDHHaXH4JvgMy0f8dnnZeduBgGy6 +s7ijNm1KDK6HjVcFUWD8+fYgovq0cQJSPTNniJuxpfqo11FzdQUABKZF3MQcQ1mi +BKJne60fcrnR1rwZfCd/lYyQYKoXnDqv2fm6mwIDAQABAoIBAAfOUXD4xJDAeNkq +liVCEUzzXu+3vEUhyaqI+KQfPmNIS/zqWNUPHBqR0YitZWVaQ3hYXhDRNLoIeFdM +6vEPBrMwHuYehl5nOcCSEK24Lw58TEuIkC7QBjmvv0+bZfe17ENsf5AoQIMJwQtL +koZNyrIc+/CSUPQ6mc4j69kb46Okb1T1BvlIJrvzAlb6RTvJD1IjCJFj7Tn5yXjF +mbCynwrOvlc5E2XQHlGbabQGf8uHIlu7P6+L4vPwP26t2nIVSY+1G9/IF0sW9PH7 +EFsdovxYlsKBAX6IdjEi65347IglmCloAcuRz1AOGys8dgPitkDNL2CMgdXuOuj1 +ONkpZ4ECgYEA9FEVK/Gpb0eHlWVJMu5bGa3yADpxUwy5xO7eOd103m39tSUXOw6E +LLb5AHcYkwsOGsz0KrsVPOLC69kQpvaxr2duIzs085x9XikxlDDLJzIfnLgVv7Ib +CSvRR3R8FSAuXxIa68Yx5rG910dcQV3nl8iFzCHcmH9ECI6NQohELisCgYEAzyAl +F8ZQbIA4LPvh8PKlBpuWc3lzqALmv64dtQvhBL3N39J1YpiyxjCgKljIKu5Qk+t/ +WLJAPAHXJCTZgzdQsmcc1U39ziUhJFVEcXfVP777+O0gQwIzLZcKGc9rOoXFlblW +7UjEGbrCQITS9Ir1iBSgfw04PUcUd5dbvWek3VECgYEAjFhTkCXHTgxJ/3Dqhp5T +qMG6ZZUs6idCQ7Vv5M+pRejrN/axjJQ/KyyEh1biv/02wgCANle30Hz2ueK8ZR0L +XxZMN2LYfSSlA8UoHNeWq9JoRG+e1rqqOy93jdOFP+F3oddVraDxo3Lw+cydW1Nl +KVTgPy4oeVWKMFwrG1AJ0ncCgYEAwPxbc051CtNhBBDg0UbUGlcHlKo55/ZkF37c +8R6TV36d/wiyFN7f26fc4/f68X1BGMHY0sSq8v5n/aZUAF2e08sdY2WasOOJgLft +4Kddy1pgnewbHjRDiRvs7mWDrHCNy9Z3tvkQtkR7z++yOuXqphNKA3dGylmbKV6e +vNiAFTECgYBAQyPzQTVSyS950qd5U87EsWKBKRRr0STW+ualYBsbDe4Mt5NlPdJm +WUhUPNkOObFZgVOJEzVYCfiuWu/XOe/PZgdG3hxtaRrRr4WTCt+T7pUhOTaTQM2C +lxTVrBYqlYMLrD+ZJtSJSRfgII4i6JBynH5aVxVjq6Hv2uZyTXcAgg== -----END RSA PRIVATE KEY----- From 4b21d67db3ba117806d31293157a920d6b047eeb Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 15 Feb 2024 17:41:50 +0100 Subject: [PATCH 71/78] [Connectors API] Check for null sync job id in UpdateConnectorSyncJobIngestionStatsActionRequestTests (#105551) --- ...orSyncJobIngestionStatsActionRequestTests.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestTests.java index 48ab14558db7e..4f78ad3ffa7e7 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestTests.java @@ -47,6 +47,21 @@ public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { assertThat(exception.getMessage(), containsString(EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); } + public void testValidate_WhenConnectorSyncJobIdIsNull_ExpectValidationError() { + UpdateConnectorSyncJobIngestionStatsAction.Request request = new UpdateConnectorSyncJobIngestionStatsAction.Request( + null, + 0L, + 0L, + 0L, + 0L, + Instant.now() + ); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } + public void testValidate_WhenDeletedDocumentCountIsNegative_ExpectValidationError() { UpdateConnectorSyncJobIngestionStatsAction.Request request = new UpdateConnectorSyncJobIngestionStatsAction.Request( randomAlphaOfLength(10), From e86c2ace46a4074f274a7c5c478eb1d7fb5794c1 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 15 Feb 2024 17:29:07 +0000 Subject: [PATCH 72/78] [ML] Alter comment that referred to RapidJSON (#105562) We no longer use RapidJSON in the ML C++, so make a comment that referred to it more generic. --- .../xpack/ml/process/StateToProcessWriterHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/StateToProcessWriterHelper.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/StateToProcessWriterHelper.java index 4c42609c44833..72395e5295d7a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/StateToProcessWriterHelper.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/StateToProcessWriterHelper.java @@ -27,8 +27,8 @@ public static void writeStateToStream(BytesReference source, OutputStream stream --length; } source.slice(0, length).writeTo(stream); - // This is dictated by RapidJSON on the C++ side; it treats a '\0' as end-of-file - // even when it's not really end-of-file, and this is what we need because we're + // This is dictated by the JSON parser on the C++ side; it treats a '\0' as the character + // that separates distinct JSON documents, and this is what we need because we're // sending multiple JSON documents via the same named pipe. stream.write(0); } From 431f4f0bfe7ec7c1e0bbdf09c7a43bac7a9ee9c0 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 18:16:06 +0000 Subject: [PATCH 73/78] Move AwaitsFix for #105543 --- .../elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java | 2 ++ .../java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java index ead0173576ff7..85eb0c02625ad 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.action; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.DocWriteResponse; @@ -34,6 +35,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105543") @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") public class EsqlActionBreakerIT extends EsqlActionIT { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index ef013749fc896..b23c75df6fa4f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -1316,7 +1316,6 @@ public void testStatsNestFields() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105543") public void testStatsMissingFieldWithStats() { final String node1, node2; if (randomBoolean()) { From 009a847273a48633e735c8934c6c994f0e9877f3 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 15 Feb 2024 11:23:19 -0700 Subject: [PATCH 74/78] Always show `composed_of` field for composable index templates (#105315) * Always show `composed_of` field for composable index templates Prior to e786cfa7061b427cf6185ad907069838dd679574 we inadvertently always added composable index templates with `composed_of: []` beacuse https://github.com/elastic/elasticsearch/commit/e786cfa7061b427cf6185ad907069838dd679574#diff-5081302eb39033199deb1977d544d1cd7867212a92b8d77e0aa0ded361272b11L618-L630 created a new `ComposableIndexTemplate` from an existing one, and the `.composedOf()` field returned an empty list of no component templates were provided: https://github.com/elastic/elasticsearch/blob/89e714ee5dc60db8b4979ab6372ff767e108e9da/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java#L172-L177 This meant that before 8.12.0 we would always show `composed_of: []` for composable index templates. This commit recreates this behavior, and always displays the empty list even if no component templates are used by a composable index template. Resolves #104627 --- docs/changelog/105315.yaml | 6 ++++++ .../test/indices.get_index_template/10_basic.yml | 13 +++++++++++++ .../cluster/metadata/ComposableIndexTemplate.java | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/105315.yaml diff --git a/docs/changelog/105315.yaml b/docs/changelog/105315.yaml new file mode 100644 index 0000000000000..207e72467a689 --- /dev/null +++ b/docs/changelog/105315.yaml @@ -0,0 +1,6 @@ +pr: 105315 +summary: Always show `composed_of` field for composable index templates +area: Indices APIs +type: bug +issues: + - 104627 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml index 41e5506c412cd..fcf4a75af2227 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml @@ -188,3 +188,16 @@ setup: type: keyword lifecycle: data_retention: "30d" + +--- +"Get index template always shows composed_of": + - skip: + version: " - 8.12.99" + reason: "A bug was fixed in 8.13.0 to make `composed_of` always returned" + + - do: + indices.get_index_template: + name: test + + - match: {index_templates.0.name: test} + - match: {index_templates.0.index_template.composed_of: []} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index dfd4431eb073c..7702ec0ac0b5c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -125,7 +125,7 @@ private ComposableIndexTemplate( ) { this.indexPatterns = indexPatterns; this.template = template; - this.componentTemplates = componentTemplates; + this.componentTemplates = componentTemplates == null ? List.of() : componentTemplates; this.priority = priority; this.version = version; this.metadata = metadata; From d8e66259bbbf2be8285adcc4f06a5cff9798e7b7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 15 Feb 2024 19:20:19 +0000 Subject: [PATCH 75/78] Fix use-after-free at event-loop shutdown (#105486) We could still be manipulating a network message when the event loop shuts down, causing us to close the message while it's still in use. This is at best going to be a little surprising to the caller, and at worst could be an outright use-after-free bug. This commit moves the double-check for a leaked promise to happen strictly after the event loop has fully terminated, so that we can be sure we've finished using it by this point. Relates #105306, #97301 --- docs/changelog/105486.yaml | 5 ++ .../http/netty4/Netty4HttpChannel.java | 15 ++--- .../http/netty4/Netty4HttpServerChannel.java | 5 +- .../transport/netty4/Netty4TcpChannel.java | 60 ++---------------- .../netty4/Netty4TcpServerChannel.java | 4 +- .../transport/netty4/Netty4Utils.java | 61 +++++++++++++++++++ .../org/elasticsearch/http/HttpChannel.java | 4 +- .../elasticsearch/rest/RestController.java | 1 + .../transport/OutboundHandler.java | 8 +++ .../transport/TransportMessage.java | 7 +++ 10 files changed, 100 insertions(+), 70 deletions(-) create mode 100644 docs/changelog/105486.yaml diff --git a/docs/changelog/105486.yaml b/docs/changelog/105486.yaml new file mode 100644 index 0000000000000..befdaec2301c6 --- /dev/null +++ b/docs/changelog/105486.yaml @@ -0,0 +1,5 @@ +pr: 105486 +summary: Fix use-after-free at event-loop shutdown +area: Network +type: bug +issues: [] diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java index 705cb02dfb246..83728b8ef73c2 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java @@ -14,12 +14,13 @@ import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.http.HttpResponse; -import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.netty4.Netty4TcpChannel; import java.net.InetSocketAddress; import java.net.SocketAddress; +import static org.elasticsearch.transport.netty4.Netty4Utils.addListener; +import static org.elasticsearch.transport.netty4.Netty4Utils.safeWriteAndFlush; + public class Netty4HttpChannel implements HttpChannel { private final Channel channel; @@ -27,18 +28,12 @@ public class Netty4HttpChannel implements HttpChannel { Netty4HttpChannel(Channel channel) { this.channel = channel; - Netty4TcpChannel.addListener(this.channel.closeFuture(), closeContext); + addListener(this.channel.closeFuture(), closeContext); } @Override public void sendResponse(HttpResponse response, ActionListener listener) { - // We need to both guard against double resolving the listener and not resolving it in case of event loop shutdown so we need to - // use #notifyOnce here until https://github.com/netty/netty/issues/8007 is resolved. - var wrapped = ActionListener.notifyOnce(listener); - channel.writeAndFlush(response, Netty4TcpChannel.addPromise(wrapped, channel)); - if (channel.eventLoop().isShutdown()) { - wrapped.onFailure(new TransportException("Cannot send HTTP response, event loop is shutting down.")); - } + safeWriteAndFlush(channel, response, listener); } @Override diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java index 7356b7fb65cbc..58dc675b76bb1 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java @@ -13,10 +13,11 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.http.HttpServerChannel; -import org.elasticsearch.transport.netty4.Netty4TcpChannel; import java.net.InetSocketAddress; +import static org.elasticsearch.transport.netty4.Netty4Utils.addListener; + public class Netty4HttpServerChannel implements HttpServerChannel { private final Channel channel; @@ -24,7 +25,7 @@ public class Netty4HttpServerChannel implements HttpServerChannel { Netty4HttpServerChannel(Channel channel) { this.channel = channel; - Netty4TcpChannel.addListener(this.channel.closeFuture(), closeContext); + addListener(this.channel.closeFuture(), closeContext); } @Override diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java index 33fdb00e7abb2..726f1293c5cf3 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java @@ -11,19 +11,19 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPromise; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Releasables; import org.elasticsearch.transport.TcpChannel; -import org.elasticsearch.transport.TransportException; import java.net.InetSocketAddress; +import static org.elasticsearch.transport.netty4.Netty4Utils.addListener; +import static org.elasticsearch.transport.netty4.Netty4Utils.safeWriteAndFlush; + public class Netty4TcpChannel implements TcpChannel { private final Channel channel; @@ -44,52 +44,6 @@ public class Netty4TcpChannel implements TcpChannel { addListener(connectFuture, connectContext); } - /** - * Adds a listener that completes the given {@link ListenableFuture} to the given {@link ChannelFuture}. - * @param channelFuture Channel future - * @param listener Listener to complete - */ - public static void addListener(ChannelFuture channelFuture, ListenableFuture listener) { - channelFuture.addListener(f -> { - if (f.isSuccess()) { - listener.onResponse(null); - } else { - Throwable cause = f.cause(); - if (cause instanceof Error) { - ExceptionsHelper.maybeDieOnAnotherThread(cause); - listener.onFailure(new Exception(cause)); - } else { - listener.onFailure((Exception) cause); - } - } - }); - } - - /** - * Creates a {@link ChannelPromise} for the given {@link Channel} and adds a listener that invokes the given {@link ActionListener} - * on its completion. - * @param listener lister to invoke - * @param channel channel - * @return write promise - */ - public static ChannelPromise addPromise(ActionListener listener, Channel channel) { - ChannelPromise writePromise = channel.newPromise(); - writePromise.addListener(f -> { - if (f.isSuccess()) { - listener.onResponse(null); - } else { - final Throwable cause = f.cause(); - ExceptionsHelper.maybeDieOnAnotherThread(cause); - if (cause instanceof Error) { - listener.onFailure(new Exception(cause)); - } else { - listener.onFailure((Exception) cause); - } - } - }); - return writePromise; - } - @Override public void close() { if (rstOnClose) { @@ -162,13 +116,7 @@ public InetSocketAddress getRemoteAddress() { @Override public void sendMessage(BytesReference reference, ActionListener listener) { - // We need to both guard against double resolving the listener and not resolving it in case of event loop shutdown so we need to - // use #notifyOnce here until https://github.com/netty/netty/issues/8007 is resolved. - var wrapped = ActionListener.notifyOnce(listener); - channel.writeAndFlush(reference, addPromise(wrapped, channel)); - if (channel.eventLoop().isShutdown()) { - wrapped.onFailure(new TransportException("Cannot send message, event loop is shutting down.")); - } + safeWriteAndFlush(channel, reference, listener); } public Channel getNettyChannel() { diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java index 1279c50133643..13f691e6e0e5e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java @@ -16,6 +16,8 @@ import java.net.InetSocketAddress; +import static org.elasticsearch.transport.netty4.Netty4Utils.addListener; + public class Netty4TcpServerChannel implements TcpServerChannel { private final Channel channel; @@ -23,7 +25,7 @@ public class Netty4TcpServerChannel implements TcpServerChannel { Netty4TcpServerChannel(Channel channel) { this.channel = channel; - Netty4TcpChannel.addListener(this.channel.closeFuture(), closeContext); + addListener(this.channel.closeFuture(), closeContext); } @Override diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java index b9986dbf00d87..1025ad11ba05e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java @@ -11,22 +11,30 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.DefaultChannelPromise; import io.netty.util.NettyRuntime; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.ImmediateEventExecutor; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.recycler.Recycler; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Booleans; +import org.elasticsearch.transport.TransportException; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; public class Netty4Utils { @@ -121,4 +129,57 @@ public static Recycler createRecycler(Settings settings) { setAvailableProcessors(EsExecutors.allocatedProcessors(settings)); return NettyAllocator.getRecycler(); } + + /** + * Calls {@link Channel#writeAndFlush} to write the given message to the given channel, but ensures that the listener is completed even + * if the event loop is concurrently shutting down since Netty does not offer this guarantee. + */ + public static void safeWriteAndFlush(Channel channel, Object message, ActionListener listener) { + // Use ImmediateEventExecutor.INSTANCE since we want to be able to complete this promise, and any waiting listeners, even if the + // channel's event loop has shut down. Normally this completion will happen on the channel's event loop anyway because the write op + // can only be completed by some network event from this point on. However... + final var promise = new DefaultChannelPromise(channel, ImmediateEventExecutor.INSTANCE); + addListener(promise, listener); + assert assertCorrectPromiseListenerThreading(channel, promise); + channel.writeAndFlush(message, promise); + if (channel.eventLoop().isShuttingDown()) { + // ... if we get here then the event loop may already have terminated, and https://github.com/netty/netty/issues/8007 means that + // we cannot know if the preceding writeAndFlush made it onto its queue before shutdown or whether it will just vanish without a + // trace, so to avoid a leak we must double-check that the final listener is completed. + channel.eventLoop().terminationFuture().addListener(ignored -> + // NB the promise executor is ImmediateEventExecutor.INSTANCE which means this call to tryFailure() will ensure its completion, + // and the completion of any waiting listeners, without forking away from the current thread. The current thread might be the + // thread that was running the event loop since that's where the terminationFuture is completed, or it might be a thread which + // called (and is still calling) safeWriteAndFlush. + promise.tryFailure(new TransportException("Cannot send network message, event loop is shutting down."))); + } + } + + private static boolean assertCorrectPromiseListenerThreading(Channel channel, Future promise) { + final var eventLoop = channel.eventLoop(); + promise.addListener(future -> { + assert eventLoop.inEventLoop() || future.cause() instanceof RejectedExecutionException || channel.eventLoop().isTerminated() + : future.cause(); + }); + return true; + } + + /** + * Subscribes the given {@link ActionListener} to the given {@link Future}. + */ + public static void addListener(Future future, ActionListener listener) { + future.addListener(f -> { + if (f.isSuccess()) { + listener.onResponse(null); + } else { + final Throwable cause = f.cause(); + ExceptionsHelper.maybeDieOnAnotherThread(cause); + if (cause instanceof Exception exception) { + listener.onFailure(exception); + } else { + listener.onFailure(new Exception(cause)); + } + } + }); + } } diff --git a/server/src/main/java/org/elasticsearch/http/HttpChannel.java b/server/src/main/java/org/elasticsearch/http/HttpChannel.java index 635e6c095c4d0..cf3dbcf9fd029 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpChannel.java +++ b/server/src/main/java/org/elasticsearch/http/HttpChannel.java @@ -20,7 +20,9 @@ public interface HttpChannel extends CloseableChannel { * completed. * * @param response to send to channel - * @param listener to execute upon send completion + * @param listener to execute upon send completion. Note that this listener is usually completed on a network thread in a context in + * which there's a risk of stack overflows if on close it calls back into the network layer in a manner that might end + * up nesting too deeply. When in doubt, dispatch any further work onto a separate thread. */ void sendResponse(HttpResponse response, ActionListener listener); diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index e3cc95b65a165..d197fe50d60d5 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -885,6 +885,7 @@ public void close() { void addChunkLength(long chunkLength) { assert chunkLength >= 0L : chunkLength; assert Transports.assertTransportThread(); // always called on the transport worker, no need for sync + assert get() != null : "already closed"; responseLength += chunkLength; } } diff --git a/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java b/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java index 3261a6c9678b5..0bb8ddeb80c5f 100644 --- a/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java +++ b/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java @@ -71,6 +71,14 @@ void setSlowLogThreshold(TimeValue slowLogThreshold) { this.slowLogThresholdMs = slowLogThreshold.getMillis(); } + /** + * Send a raw message over the given channel. + * + * @param listener completed when the message has been sent, on the network thread (unless the network thread has shut down). Take care + * if calling back into the network layer from this listener without dispatching to a new thread since if we do that + * too many times in a row it can cause a stack overflow. When in doubt, dispatch any follow-up work onto a separate + * thread. + */ void sendBytes(TcpChannel channel, BytesReference bytes, ActionListener listener) { internalSend(channel, bytes, null, listener); } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportMessage.java b/server/src/main/java/org/elasticsearch/transport/TransportMessage.java index d5257bf2840f3..f07cce5a731f8 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportMessage.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportMessage.java @@ -51,6 +51,13 @@ public boolean tryIncRef() { return true; } + /** + * {@inheritDoc} + * + * Note that the lifetime of an outbound {@link TransportMessage} lasts at least until it has been fully sent over the network, and it + * may be closed on a network thread in a context in which there's a risk of stack overflows if on close it calls back into the network + * layer in a manner that might end up nesting too deeply. When in doubt, dispatch any further work onto a separate thread. + */ @Override public boolean decRef() { // noop, override to manage the life-cycle of resources held by a transport message From 24d7043828317f8a23b644d04f1e1945c9409e0f Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Thu, 15 Feb 2024 15:30:22 -0500 Subject: [PATCH 76/78] Update skip version on rule_query tests (#105573) * Update skip version on rule_query tests * Update 260_rule_query_search.yml --- .../rest-api-spec/test/entsearch/260_rule_query_search.yml | 2 +- .../xpack/ml/queries/TextExpansionQueryBuilder.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml index 40cdb7839c9ed..f67c955126235 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml @@ -239,7 +239,7 @@ setup: --- "Perform a rule query with an organic query that must be rewritten to another query type": - skip: - version: " - 8.13.99" + version: " - 8.12.1" reason: Bugfix that was broken in previous versions - do: diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java index a392996fbb448..7d5197b9e9ba0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java @@ -140,6 +140,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (weightedTokensSupplier != null) { if (weightedTokensSupplier.get() == null) { return this; From 7870c3a53fcdc17b9d26c5e55c1efce86445e20d Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 16 Feb 2024 03:54:17 +0100 Subject: [PATCH 77/78] Save needless TreeMap.contains in TimeSeriesIdFieldMapper (#105583) We were burning quite a few cycles here on traversing the tree map twice. --- .../elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 6fe39232013d3..fb26debab2acc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -345,11 +345,9 @@ public DocumentDimensions validate(final IndexSettings settings) { } private void add(String fieldName, BytesReference encoded) throws IOException { - final Dimension dimension = new Dimension(new BytesRef(fieldName), encoded); - if (dimensions.contains(dimension)) { + if (dimensions.add(new Dimension(new BytesRef(fieldName), encoded)) == false) { throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); } - dimensions.add(dimension); } } From 50c57e01fbfe288ee2b96de2a1e7dac79c635229 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 16 Feb 2024 05:48:23 +0100 Subject: [PATCH 78/78] Two misc. memory savings in TsidExtractingIdFieldMapper (#105584) No need for a capturing lambda here. Also no need to needlessly setup and ArrayList for the field when we only care about the first value for the name. --- .../mapper/TsidExtractingIdFieldMapper.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 27e281ed7fb52..08735911c2955 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -26,7 +26,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Locale; /** @@ -107,13 +106,13 @@ private TsidExtractingIdFieldMapper() { private static final long SEED = 0; public static void createField(DocumentParserContext context, IndexRouting.ExtractFromSource.Builder routingBuilder, BytesRef tsid) { - List timestampFields = context.rootDoc().getFields(DataStreamTimestampFieldMapper.DEFAULT_PATH); - if (timestampFields.isEmpty()) { + final IndexableField timestampField = context.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); + if (timestampField == null) { throw new IllegalArgumentException( "data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] is missing" ); } - long timestamp = timestampFields.get(0).numericValue().longValue(); + long timestamp = timestampField.numericValue().longValue(); byte[] suffix = new byte[16]; String id = createId(context.hasDynamicMappers() == false, routingBuilder, tsid, timestamp, suffix); /* @@ -160,14 +159,11 @@ public static String createId( ByteUtils.writeLongLE(hash.h1, suffix, 0); ByteUtils.writeLongBE(timestamp, suffix, 8); // Big Ending shrinks the inverted index by ~37% - String id = routingBuilder.createId(suffix, () -> { - if (dynamicMappersExists == false) { - throw new IllegalStateException( - "Didn't find any fields to include in the routing which would be fine if there are" - + " dynamic mapping waiting but we couldn't find any of those either!" - ); - } - return 0; + String id = routingBuilder.createId(suffix, dynamicMappersExists ? () -> 0 : () -> { + throw new IllegalStateException( + "Didn't find any fields to include in the routing which would be fine if there are" + + " dynamic mapping waiting but we couldn't find any of those either!" + ); }); assert Uid.isURLBase64WithoutPadding(id); // Make sure we get to use Uid's nice optimizations return id;