diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 235a4b2dbb4ad..ea4f34bcbe11e 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -8,6 +8,7 @@ "admin", "write" ], + "allowed_list": ["elastic-renovate-prod[bot]"], "set_commit_status": false, "build_on_commit": true, "build_on_comment": true, diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 9d78d3229edc1..d80256ee36a17 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -27,7 +27,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:277ebb42c458ef39cb4028f9204f0b3d51d8cd628ea737a65696a1143c3e42fe", "-wolfi", "apk" ), diff --git a/docs/changelog/113374.yaml b/docs/changelog/113374.yaml new file mode 100644 index 0000000000000..f1d5750de0f60 --- /dev/null +++ b/docs/changelog/113374.yaml @@ -0,0 +1,5 @@ +pr: 113374 +summary: Add ESQL match function +area: ES|QL +type: feature +issues: [] diff --git a/docs/changelog/113735.yaml b/docs/changelog/113735.yaml new file mode 100644 index 0000000000000..4f6579c7cb9e0 --- /dev/null +++ b/docs/changelog/113735.yaml @@ -0,0 +1,28 @@ +pr: 113735 +summary: "ESQL: Introduce per agg filter" +area: ES|QL +type: feature +issues: [] +highlight: + title: "ESQL: Introduce per agg filter" + body: |- + Add support for aggregation scoped filters that work dynamically on the + data in each group. + + [source,esql] + ---- + | STATS success = COUNT(*) WHERE 200 <= code AND code < 300, + redirect = COUNT(*) WHERE 300 <= code AND code < 400, + client_err = COUNT(*) WHERE 400 <= code AND code < 500, + server_err = COUNT(*) WHERE 500 <= code AND code < 600, + total_count = COUNT(*) + ---- + + Implementation wise, the base AggregateFunction has been extended to + allow a filter to be passed on. This is required to incorporate the + filter as part of the aggregate equality/identity which would fail with + the filter as an external component. + As part of the process, the serialization for the existing aggregations + had to be fixed so AggregateFunction implementations so that it + delegates to their parent first. + notable: true diff --git a/docs/changelog/114168.yaml b/docs/changelog/114168.yaml new file mode 100644 index 0000000000000..58f1ab7110e7d --- /dev/null +++ b/docs/changelog/114168.yaml @@ -0,0 +1,5 @@ +pr: 114168 +summary: Add a query rules tester API call +area: Relevance +type: enhancement +issues: [] diff --git a/docs/changelog/114407.yaml b/docs/changelog/114407.yaml new file mode 100644 index 0000000000000..4c1134a9d3834 --- /dev/null +++ b/docs/changelog/114407.yaml @@ -0,0 +1,6 @@ +pr: 114407 +summary: Fix synthetic source handling for `bit` type in `dense_vector` field +area: Search +type: bug +issues: + - 114402 diff --git a/docs/changelog/114439.yaml b/docs/changelog/114439.yaml new file mode 100644 index 0000000000000..fd097d02f885f --- /dev/null +++ b/docs/changelog/114439.yaml @@ -0,0 +1,5 @@ +pr: 114439 +summary: Adding new bbq index types behind a feature flag +area: Vector Search +type: feature +issues: [] diff --git a/docs/changelog/114453.yaml b/docs/changelog/114453.yaml new file mode 100644 index 0000000000000..0d5345ad9d2a6 --- /dev/null +++ b/docs/changelog/114453.yaml @@ -0,0 +1,5 @@ +pr: 114453 +summary: Switch default chunking strategy to sentence +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/114482.yaml b/docs/changelog/114482.yaml new file mode 100644 index 0000000000000..a5e2e981f7adc --- /dev/null +++ b/docs/changelog/114482.yaml @@ -0,0 +1,5 @@ +pr: 114482 +summary: Remove snapshot build restriction for match and qstr functions +area: ES|QL +type: feature +issues: [] diff --git a/docs/changelog/114549.yaml b/docs/changelog/114549.yaml new file mode 100644 index 0000000000000..a6bdbba93876b --- /dev/null +++ b/docs/changelog/114549.yaml @@ -0,0 +1,5 @@ +pr: 114549 +summary: Send mid-stream errors to users +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/114596.yaml b/docs/changelog/114596.yaml new file mode 100644 index 0000000000000..a36978dcacd8c --- /dev/null +++ b/docs/changelog/114596.yaml @@ -0,0 +1,5 @@ +pr: 114596 +summary: Stream Google Completion +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/114623.yaml b/docs/changelog/114623.yaml new file mode 100644 index 0000000000000..817a8e874bcc0 --- /dev/null +++ b/docs/changelog/114623.yaml @@ -0,0 +1,5 @@ +pr: 114623 +summary: Preserve thread context when waiting for segment generation in RTG +area: CRUD +type: bug +issues: [] diff --git a/docs/changelog/114636.yaml b/docs/changelog/114636.yaml new file mode 100644 index 0000000000000..c63876fda67f7 --- /dev/null +++ b/docs/changelog/114636.yaml @@ -0,0 +1,5 @@ +pr: 114636 +summary: Dynamically get of num allocations +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/114638.yaml b/docs/changelog/114638.yaml new file mode 100644 index 0000000000000..0386aacfe3e18 --- /dev/null +++ b/docs/changelog/114638.yaml @@ -0,0 +1,7 @@ +pr: 114638 +summary: "ES|QL: Restrict sorting for `_source` and counter field types" +area: ES|QL +type: bug +issues: + - 114423 + - 111976 diff --git a/docs/changelog/114683.yaml b/docs/changelog/114683.yaml new file mode 100644 index 0000000000000..a677e65a12b0e --- /dev/null +++ b/docs/changelog/114683.yaml @@ -0,0 +1,5 @@ +pr: 114683 +summary: Default inference endpoint for the multilingual-e5-small model +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/114715.yaml b/docs/changelog/114715.yaml new file mode 100644 index 0000000000000..0894cb2fa42ca --- /dev/null +++ b/docs/changelog/114715.yaml @@ -0,0 +1,5 @@ +pr: 114715 +summary: Ignore unrecognized openai sse fields +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/114719.yaml b/docs/changelog/114719.yaml new file mode 100644 index 0000000000000..477d656d5b979 --- /dev/null +++ b/docs/changelog/114719.yaml @@ -0,0 +1,5 @@ +pr: 114719 +summary: Wait for allocation on scale up +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/114732.yaml b/docs/changelog/114732.yaml new file mode 100644 index 0000000000000..42176cdbda443 --- /dev/null +++ b/docs/changelog/114732.yaml @@ -0,0 +1,5 @@ +pr: 114732 +summary: Stream Bedrock Completion +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/reference/data-streams/set-up-tsds.asciidoc b/docs/reference/data-streams/set-up-tsds.asciidoc index 3a483ac351180..d082a9c4eebeb 100644 --- a/docs/reference/data-streams/set-up-tsds.asciidoc +++ b/docs/reference/data-streams/set-up-tsds.asciidoc @@ -121,7 +121,8 @@ naming scheme]. * Specify a mapping that defines your dimensions and metrics: ** One or more <> with a `time_series_dimension` value of `true`. - At least one of these dimensions must be a plain `keyword` field. + Alternatively, one or more <> fields configured as dimension containers, + provided that they will contain at least one sub-field (mapped statically or dynamically). ** One or more <>, marked using the `time_series_metric` mapping parameter. @@ -203,10 +204,9 @@ DELETE _ilm/policy/my-weather-sensor-lifecycle-policy Documents in a TSDS must include: * A `@timestamp` field -* One or more dimension fields. At least one dimension must be a `keyword` field -that matches the `index.routing_path` index setting, if specified. If not specified -explicitly, `index.routing_path` is set automatically to whichever mappings have - `time_series_dimension` set to `true`. +* One or more dimension fields. At least one dimension must match the `index.routing_path` index setting, +if specified. If not specified explicitly, `index.routing_path` is set automatically to whichever mappings have +`time_series_dimension` set to `true`. To automatically create your TSDS, submit an indexing request that targets the TSDS's name. This name must match one of your index template's @@ -285,13 +285,12 @@ POST metrics-weather_sensors-dev/_rollover Configuring a TSDS via an index template that uses component templates is a bit more complicated. Typically with component templates mappings and settings get scattered across multiple component templates. -When configuring the `index.mode` setting in a component template, the `index.routing_path` setting needs to -be defined in the same component template. Additionally the fields mentioned in the `index.routing_path` -also need to be defined in the same component template with the `time_series_dimension` attribute enabled. +If the `index.routing_path` is defined, the fields it references need to be defined in the same component +template with the `time_series_dimension` attribute enabled. -The reasons for this is that each component template needs to be valid on its own and the time series index mode -requires the `index.routing_path` setting. When configuring the `index.mode` setting in an index template, the `index.routing_path` setting is configured automatically. It is derived from -the field mappings with `time_series_dimension` attribute enabled. +The reasons for this is that each component template needs to be valid on its own. When configuring the +`index.mode` setting in an index template, the `index.routing_path` setting is configured automatically. +It is derived from the field mappings with `time_series_dimension` attribute enabled. [discrete] [[set-up-tsds-whats-next]] diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 01573658c33d0..461c0a1272e96 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -109,7 +109,10 @@ parameter: * <> * <> -For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. For details refer to <>. +For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. +For details refer to <>. + +Dimension definitions can be simplified through <> fields. [discrete] [[time-series-metric]] @@ -294,12 +297,15 @@ When you create the matching index template for a TSDS, you must specify one or more dimensions in the `index.routing_path` setting. Each document in a TSDS must contain one or more dimensions that match the `index.routing_path` setting. -Dimensions in the `index.routing_path` setting must be plain `keyword` fields. The `index.routing_path` setting accepts wildcard patterns (for example `dim.*`) and can dynamically match new fields. However, {es} will reject any mapping -updates that add scripted, runtime, or non-dimension, non-`keyword` fields that +updates that add scripted, runtime, or non-dimension fields that match the `index.routing_path` value. +<> fields may be configured +as dimension containers. In this case, their sub-fields get included to the +routing path automatically. + TSDS documents don't support a custom `_routing` value. Similarly, you can't require a `_routing` value in mappings for a TSDS. diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc new file mode 100644 index 0000000000000..2a27fe4814395 --- /dev/null +++ b/docs/reference/esql/functions/description/match.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a match query on the specified field. Returns true if the provided query matches the row. diff --git a/docs/reference/esql/functions/examples/match.asciidoc b/docs/reference/esql/functions/examples/match.asciidoc new file mode 100644 index 0000000000000..3f31d68ea9abb --- /dev/null +++ b/docs/reference/esql/functions/examples/match.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/match-function.csv-spec[tag=match-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/match-function.csv-spec[tag=match-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json new file mode 100644 index 0000000000000..8a355360a790f --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -0,0 +1,85 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "match", + "description" : "Performs a match query on the specified field. Returns true if the provided query matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "from books \n| where match(author, \"Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;" + ], + "preview" : true, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json index 72be906cbae63..9823c3cff8923 100644 --- a/docs/reference/esql/functions/kibana/definition/qstr.json +++ b/docs/reference/esql/functions/kibana/definition/qstr.json @@ -33,5 +33,5 @@ "from books \n| where qstr(\"author: Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;" ], "preview" : true, - "snapshot_only" : true + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md new file mode 100644 index 0000000000000..3c06662982bbf --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/match.md @@ -0,0 +1,14 @@ + + +### MATCH +Performs a match query on the specified field. Returns true if the provided query matches the row. + +``` +from books +| where match(author, "Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +``` diff --git a/docs/reference/esql/functions/layout/match.asciidoc b/docs/reference/esql/functions/layout/match.asciidoc new file mode 100644 index 0000000000000..e62c81548c2b1 --- /dev/null +++ b/docs/reference/esql/functions/layout/match.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-match]] +=== `MATCH` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/match.svg[Embedded,opts=inline] + +include::../parameters/match.asciidoc[] +include::../description/match.asciidoc[] +include::../types/match.asciidoc[] +include::../examples/match.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/match.asciidoc b/docs/reference/esql/functions/parameters/match.asciidoc new file mode 100644 index 0000000000000..f18adb28cd20c --- /dev/null +++ b/docs/reference/esql/functions/parameters/match.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Field that the query will target. + +`query`:: +Text you wish to find in the provided field. diff --git a/docs/reference/esql/functions/signature/match.svg b/docs/reference/esql/functions/signature/match.svg new file mode 100644 index 0000000000000..e7bb001247a9d --- /dev/null +++ b/docs/reference/esql/functions/signature/match.svg @@ -0,0 +1 @@ +MATCH(field,query) diff --git a/docs/reference/esql/functions/types/match.asciidoc b/docs/reference/esql/functions/types/match.asciidoc new file mode 100644 index 0000000000000..7523b29c62b1d --- /dev/null +++ b/docs/reference/esql/functions/types/match.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | query | result +keyword | keyword | boolean +keyword | text | boolean +text | keyword | boolean +text | text | boolean +|=== diff --git a/docs/reference/esql/processing-commands/where.asciidoc b/docs/reference/esql/processing-commands/where.asciidoc index 407df30c57215..1d6fc1e90d595 100644 --- a/docs/reference/esql/processing-commands/where.asciidoc +++ b/docs/reference/esql/processing-commands/where.asciidoc @@ -5,6 +5,13 @@ The `WHERE` processing command produces a table that contains all the rows from the input table for which the provided condition evaluates to `true`. +[TIP] +==== +In case of value exclusions, fields with `null` values will be excluded from search results. +In this context a `null` means either there is an explicit `null` value in the document or there is no value at all. +For example: `WHERE field != "value"` will be interpreted as `WHERE field != "value" AND field IS NOT NULL`. +==== + **Syntax** [source,esql] diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index ed8cf6c1494e4..1c8f1db216b75 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -122,7 +122,7 @@ preview:[] The number of shards a custom <> value can go to. Defaults to 1 and can only be set at index creation time. This value must be less - than the `index.number_of_shards` unless the `index.number_of_shards` value is also 1. + than the `index.number_of_routing_shards` unless the `index.number_of_routing_shards` value is also 1. See <> for more details about how this setting is used. [[ccr-index-soft-deletes]] diff --git a/docs/reference/mapping/params/subobjects.asciidoc b/docs/reference/mapping/params/subobjects.asciidoc index b0a5d3817c332..ff91f07cfb359 100644 --- a/docs/reference/mapping/params/subobjects.asciidoc +++ b/docs/reference/mapping/params/subobjects.asciidoc @@ -111,6 +111,7 @@ PUT my-index-000001/_doc/metric_1 The `subobjects` setting for existing fields and the top-level mapping definition cannot be updated. +[[subobjects-auto-flattening]] ==== Auto-flattening object mappings It is generally recommended to define the properties of an object that is configured with `subobjects: false` with dotted field names diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 7e2e7083fa70b..babe4f508b5f0 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -35,12 +35,13 @@ Dates:: Date types, including <> and [[object-types]] ==== Objects and relational types -<>:: A JSON object. -<>:: An entire JSON object as a single field value. -<>:: A JSON object that preserves the relationship - between its subfields. -<>:: Defines a parent/child relationship for documents - in the same index. +<>:: A JSON object. +<>:: An entire JSON object as a single field value. +<>:: A JSON object that preserves the relationship + between its subfields. +<>:: Defines a parent/child relationship for documents + in the same index. +<>:: Provides aliases for sub-fields at the same level. [discrete] @@ -167,6 +168,8 @@ include::types/numeric.asciidoc[] include::types/object.asciidoc[] +include::types/passthrough.asciidoc[] + include::types/percolator.asciidoc[] include::types/point.asciidoc[] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 0cd9ee0578b70..44f90eded8632 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -115,22 +115,27 @@ that sacrifices result accuracy for improved speed. ==== Automatically quantize vectors for kNN search The `dense_vector` type supports quantization to reduce the memory footprint required when <> `float` vectors. -The two following quantization strategies are supported: +The three following quantization strategies are supported: + -- -`int8` - Quantizes each dimension of the vector to 1-byte integers. This can reduce the memory footprint by 75% at the cost of some accuracy. -`int4` - Quantizes each dimension of the vector to half-byte integers. This can reduce the memory footprint by 87% at the cost of some accuracy. +`int8` - Quantizes each dimension of the vector to 1-byte integers. This reduces the memory footprint by 75% (or 4x) at the cost of some accuracy. +`int4` - Quantizes each dimension of the vector to half-byte integers. This reduces the memory footprint by 87% (or 8x) at the cost of accuracy. +`bbq` - experimental:[] Better binary quantization which reduces each dimension to a single bit precision. This reduces the memory footprint by 96% (or 32x) at a larger cost of accuracy. Generally, oversampling during query time and reranking can help mitigate the accuracy loss. -- -To use a quantized index, you can set your index type to `int8_hnsw` or `int4_hnsw`. When indexing `float` vectors, the current default +When using a quantized format, you may want to oversample and rescore the results to improve accuracy. See <> for more information. + +To use a quantized index, you can set your index type to `int8_hnsw`, `int4_hnsw`, or `bbq_hnsw`. When indexing `float` vectors, the current default index type is `int8_hnsw`. NOTE: Quantization will continue to keep the raw float vector values on disk for reranking, reindexing, and quantization improvements over the lifetime of the data. -This means disk usage will increase by ~25% for `int8` and ~12.5% for `int4` due to the overhead of storing the quantized and raw vectors. +This means disk usage will increase by ~25% for `int8`, ~12.5% for `int4`, and ~3.1% for `bbq` due to the overhead of storing the quantized and raw vectors. NOTE: `int4` quantization requires an even number of vector dimensions. +NOTE: experimental:[] `bbq` quantization only supports vector dimensions that are greater than 64. + Here is an example of how to create a byte-quantized index: [source,console] @@ -173,6 +178,27 @@ PUT my-byte-quantized-index } -------------------------------------------------- +experimental:[] Here is an example of how to create a binary quantized index: + +[source,console] +-------------------------------------------------- +PUT my-byte-quantized-index +{ + "mappings": { + "properties": { + "my_vector": { + "type": "dense_vector", + "dims": 64, + "index": true, + "index_options": { + "type": "bbq_hnsw" + } + } + } + } +} +-------------------------------------------------- + [role="child_attributes"] [[dense-vector-params]] ==== Parameters for dense vector fields @@ -301,11 +327,16 @@ by 4x at the cost of some accuracy. See <>. +* experimental:[] `bbq_hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically binary +quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint +by 32x at the cost of accuracy. See <>. * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values. * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports `element_type` of `float`. * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports `element_type` of `float`. +* experimental:[] `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatically binary quantization. Only supports +`element_type` of `float`. -- `m`::: (Optional, integer) diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc new file mode 100644 index 0000000000000..f4f1945d21537 --- /dev/null +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -0,0 +1,218 @@ +[[passthrough]] +=== Pass-through object field type +++++ +Pass-through object +++++ + +Pass-through objects extend the functionality of <> by allowing to access +their subfields without including the name of the pass-through object as prefix. For instance: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", <1> + "priority": 10, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } +} + +PUT my-index-000001/_doc/1 +{ + "attributes" : { <2> + "id": "foo", + "zone": 10 + } +} + +GET my-index-000001/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "id": "foo" }}, <3> + { "match": { "zone": 10 }} + ] + } + } +} + +GET my-index-000001/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "attributes.id": "foo" }}, <4> + { "match": { "attributes.zone": 10 }} + ] + } + } +} + +-------------------------------------------------- + +<1> An object is defined as pass-through. Its priority (required) is used for conflict resolution. +<2> Object contents get indexed as usual, including dynamic mappings. +<3> Sub-fields can be referenced in queries as if they're defined at the root level. +<4> Sub-fields can also be referenced including the object name as prefix. + +[[passthrough-conflicts]] +==== Conflict resolution + +It's possible for conflicting names to arise, for fields that are defined within different scopes: + + a. A pass-through object is defined next to a field that has the same name as one of the pass-through object +sub-fields, e.g. ++ +[source,console] +-------------------------------------------------- +PUT my-index-000001/_doc/1 +{ + "attributes" : { + "id": "foo" + }, + "id": "bar" +} +-------------------------------------------------- ++ +In this case, references to `id` point to the field at the root level, while field `attributes.id` +can only be accessed using the full path. + + b. Two (or more) pass-through objects are defined within the same object and contain fields with the same name, e.g. ++ +[source,console] +-------------------------------------------------- +PUT my-index-000002 +{ + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", + "priority": 10, + "properties": { + "id": { + "type": "keyword" + } + } + }, + "resource.attributes": { + "type": "passthrough", + "priority": 20, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } +} +-------------------------------------------------- ++ +In this case, param `priority` is used for conflict resolution, with the higher values taking precedence. In the +example above, `resource.attributes` has higher priority than `attributes`, so references to `id` point to the field +within `resource.attributes`. `attributes.id` can still be accessed using its full path. + +[[passthrough-dimensions]] +==== Defining sub-fields as time-series dimensions + +It is possible to configure a pass-through field as a container for <>. +In this case, all sub-fields get annotated with the same parameter under the covers, and they're also +included in <> and <> calculations, thus simplifying +the <> setup: + +[source,console] +-------------------------------------------------- +PUT _index_template/my-metrics +{ + "index_patterns": ["metrics-mymetrics-*"], + "priority": 200, + "data_stream": { }, + "template": { + "settings": { + "index.mode": "time_series" + }, + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", + "priority": 10, + "time_series_dimension": true, + "properties": { + "host.name": { + "type": "keyword" + } + } + }, + "cpu": { + "type": "integer", + "time_series_metric": "counter" + } + } + } + } +} + +POST metrics-mymetrics-test/_doc +{ + "@timestamp": "2020-01-01T00:00:00.000Z", + "attributes" : { + "host.name": "foo", + "zone": "bar" + }, + "cpu": 10 +} +-------------------------------------------------- +// TEST[skip: The @timestamp value won't match an accepted range in the TSDS] + +In the example above, `attributes` is defined as a dimension container. Its sub-fields `host.name` (static) and `zone` +(dynamic) get included in the routing path and tsid, and can be referenced in queries without the `attributes.` prefix. + +[[passthrough-flattening]] +==== Sub-field auto-flattening + +Pass-through fields apply <> to sub-fields by default, to reduce dynamic +mapping conflicts. As a consequence, no sub-object definitions are allowed within pass-through fields. + +[[passthrough-params]] +==== Parameters for `passthrough` fields + +The following parameters are accepted by `passthrough` fields: + +[horizontal] + +<>:: + + (Required) used for naming conflict resolution between pass-through fields. The field with the highest value wins. + Accepts non-negative integer values. + +<>:: + + Whether or not to treat sub-fields as <>. + Accepts `false` (default) or `true`. + +<>:: + + Whether or not new `properties` should be added dynamically to an existing object. + Accepts `true` (default), `runtime`, `false` and `strict`. + +<>:: + + Whether the JSON value given for the object field should be parsed and indexed (`true`, default) + or completely ignored (`false`). + +<>:: + + The fields within the object, which can be of any <>, including `object`. + New properties may be added to an existing object. + +IMPORTANT: If you need to index arrays of objects instead of single objects, read <> first. diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 8fdc9f2e4f9cb..1e4c5a21d386c 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -153,23 +153,34 @@ The only requirements are that each node must be: * Accessible at its transport publish address by all other nodes in its cluster, and by any remote clusters that will discover it using - <>. + <>. Each node must have its own distinct publish address. If you specify the transport publish address using a hostname then {es} will resolve this hostname to an IP address once during startup, and other nodes will use the resulting IP address instead of resolving the name again -themselves. To avoid confusion, use a hostname which resolves to the node's -address in all network locations. +themselves. You must use a hostname such that all of the addresses to which it +resolves are addresses at which the node is accessible from all other nodes. To +avoid confusion, it is simplest to use a hostname which resolves to a single +address. + +If you specify the transport publish address using a +<> then {es} will resolve this value to +a single IP address during startup, and other nodes will use the resulting IP +address instead of resolving the value again themselves. You must use a value +such that all of the addresses to which it resolves are addresses at which the +node is accessible from all other nodes. To avoid confusion, it is simplest to +use a value which resolves to a single address. It is usually a mistake to use +`0.0.0.0` as a publish address on hosts with more than one network interface. ===== Using a single address The most common configuration is for {es} to bind to a single address at which -it is accessible to clients and other nodes. In this configuration you should -just set `network.host` to that address. You should not separately set any bind -or publish addresses, nor should you separately configure the addresses for the -HTTP or transport interfaces. +it is accessible to clients and other nodes. To use this configuration, set +only `network.host` to the desired address. Do not separately set any bind or +publish addresses. Do not separately specify the addresses for the HTTP or +transport interfaces. ===== Using multiple addresses diff --git a/docs/reference/query-rules/apis/index.asciidoc b/docs/reference/query-rules/apis/index.asciidoc index 53d5fc3dc4eee..fbeb477acacb5 100644 --- a/docs/reference/query-rules/apis/index.asciidoc +++ b/docs/reference/query-rules/apis/index.asciidoc @@ -23,6 +23,7 @@ Use the following APIs to manage query rulesets: * <> * <> * <> +* preview:[] <> include::put-query-ruleset.asciidoc[] include::get-query-ruleset.asciidoc[] @@ -31,4 +32,5 @@ include::delete-query-ruleset.asciidoc[] include::put-query-rule.asciidoc[] include::get-query-rule.asciidoc[] include::delete-query-rule.asciidoc[] +include::test-query-ruleset.asciidoc[] diff --git a/docs/reference/query-rules/apis/test-query-ruleset.asciidoc b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc new file mode 100644 index 0000000000000..4a670645cea6e --- /dev/null +++ b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc @@ -0,0 +1,133 @@ +[role="xpack"] +[[test-query-ruleset]] +=== Test query ruleset + +++++ +Tests query ruleset +++++ + +Evaluates match criteria against a query ruleset to identify the rules that would match that criteria. + +preview::[] + +[[test-query-ruleset-request]] +==== {api-request-title} + +`POST _query_rules//_test` + +[[test-query-ruleset-prereq]] +==== {api-prereq-title} + +Requires the `manage_search_query_rules` privilege. + +[[test-query-ruleset-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[test-query-rule-request-body]] +==== {api-request-body-title} + +`match_criteria`:: +(Required, object) Defines the match criteria to apply to rules in the given query ruleset. +Match criteria should match the keys defined in the `criteria.metadata` field of the rule. + +[[test-query-ruleset-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `ruleset_id` or `match_criteria` were not provided. + +`404` (Missing resources):: +No query ruleset matching `ruleset_id` could be found. + +[[test-query-ruleset-example]] +==== {api-examples-title} + +To test a ruleset, provide the match criteria that you want to test against: + +//// + +[source,console] +-------------------------------------------------- +PUT _query_rules/my-ruleset +{ + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "contains", + "metadata": "query_string", + "values": [ "pugs", "puggles" ] + } + ], + "actions": { + "ids": [ + "id1", + "id2" + ] + } + }, + { + "rule_id": "my-rule2", + "type": "pinned", + "criteria": [ + { + "type": "fuzzy", + "metadata": "query_string", + "values": [ "rescue dogs" ] + } + ], + "actions": { + "docs": [ + { + "_index": "index1", + "_id": "id3" + }, + { + "_index": "index2", + "_id": "id4" + } + ] + } + } + ] +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _query_rules/my-ruleset +-------------------------------------------------- +// TEARDOWN + +//// + +[source,console] +---- +POST _query_rules/my-ruleset/_test +{ + "match_criteria": { + "query_string": "puggles" + } +} +---- + +A sample response: + +[source,console-result] +---- +{ + "total_matched_rules": 1, + "matched_rules": [ + { + "ruleset_id": "my-ruleset", + "rule_id": "my-rule1" + } + ] +} +---- diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 957f57ffc9105..5fd2304ff9378 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -210,7 +210,12 @@ GET /_xpack/usage "service": "elasticsearch", "task_type": "SPARSE_EMBEDDING", "count": 1 - } + }, + { + "service": "elasticsearch", + "task_type": "TEXT_EMBEDDING", + "count": 1 + }, ] }, "logstash" : { diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc index 70cf9eec121d7..6fb7f1747051f 100644 --- a/docs/reference/search/search-your-data/knn-search.asciidoc +++ b/docs/reference/search/search-your-data/knn-search.asciidoc @@ -1149,3 +1149,95 @@ POST product-index/_search ---- //TEST[continued] +[discrete] +[[dense-vector-knn-search-reranking]] +==== Oversampling and rescoring for quantized vectors + +All forms of quantization will result in some accuracy loss and as the quantization level increases the accuracy loss will also increase. +Generally, we have found that: +- `int8` requires minimal if any rescoring +- `int4` requires some rescoring for higher accuracy and larger recall scenarios. Generally, oversampling by 1.5x-2x recovers most of the accuracy loss. +- `bbq` requires rescoring except on exceptionally large indices or models specifically designed for quantization. We have found that between 3x-5x oversampling is generally sufficient. But for fewer dimensions or vectors that do not quantize well, higher oversampling may be required. + +There are two main ways to oversample and rescore. The first is to utilize the <> in the `_search` request. + +Here is an example using the top level `knn` search with oversampling and using `rescore` to rerank the results: + +[source,console] +-------------------------------------------------- +POST /my-index/_search +{ + "size": 10, <1> + "knn": { + "query_vector": [0.04283529, 0.85670587, -0.51402352, 0], + "field": "my_int4_vector", + "k": 20, <2> + "num_candidates": 50 + }, + "rescore": { + "window_size": 20, <3> + "query": { + "rescore_query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "(dotProduct(params.queryVector, 'my_int4_vector') + 1.0)", <4> + "params": { + "queryVector": [0.04283529, 0.85670587, -0.51402352, 0] + } + } + } + }, + "query_weight": 0, <5> + "rescore_query_weight": 1 <6> + } + } +} +-------------------------------------------------- +// TEST[skip: setup not provided] +<1> The number of results to return, note its only 10 and we will oversample by 2x, gathering 20 nearest neighbors. +<2> The number of results to return from the KNN search. This will do an approximate KNN search with 50 candidates +per HNSW graph and use the quantized vectors, returning the 20 most similar vectors +according to the quantized score. Additionally, since this is the top-level `knn` object, the global top 20 results +will from all shards will be gathered before rescoring. Combining with `rescore`, this is oversampling by `2x`, meaning +gathering 20 nearest neighbors according to quantized scoring and rescoring with higher fidelity float vectors. +<3> The number of results to rescore, if you want to rescore all results, set this to the same value as `k` +<4> The script to rescore the results. Script score will interact directly with the originally provided float32 vector. +<5> The weight of the original query, here we simply throw away the original score +<6> The weight of the rescore query, here we only use the rescore query + +The second way is to score per shard with the <> and <>. Generally, this means that there will be more rescoring per shard, but this +can increase overall recall at the cost of compute. + +[source,console] +-------------------------------------------------- +POST /my-index/_search +{ + "size": 10, <1> + "query": { + "script_score": { + "query": { + "knn": { <2> + "query_vector": [0.04283529, 0.85670587, -0.51402352, 0], + "field": "my_int4_vector", + "num_candidates": 20 <3> + } + }, + "script": { + "source": "(dotProduct(params.queryVector, 'my_int4_vector') + 1.0)", <4> + "params": { + "queryVector": [0.04283529, 0.85670587, -0.51402352, 0] + } + } + } + } +} +-------------------------------------------------- +// TEST[skip: setup not provided] +<1> The number of results to return +<2> The `knn` query to perform the initial search, this is executed per-shard +<3> The number of candidates to use for the initial approximate `knn` search. This will search using the quantized vectors +and return the top 20 candidates per shard to then be scored +<4> The script to score the results. Script score will interact directly with the originally provided float32 vector. diff --git a/docs/reference/search/search-your-data/semantic-search.asciidoc b/docs/reference/search/search-your-data/semantic-search.asciidoc index 62e41b3eef3de..0ef8591e42b5d 100644 --- a/docs/reference/search/search-your-data/semantic-search.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search.asciidoc @@ -104,6 +104,7 @@ IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer include::semantic-search-semantic-text.asciidoc[] +include::semantic-text-hybrid-search[] include::semantic-search-inference.asciidoc[] include::semantic-search-elser.asciidoc[] include::cohere-es.asciidoc[] diff --git a/docs/reference/search/search-your-data/semantic-text-hybrid-search b/docs/reference/search/search-your-data/semantic-text-hybrid-search new file mode 100644 index 0000000000000..c56b283434df5 --- /dev/null +++ b/docs/reference/search/search-your-data/semantic-text-hybrid-search @@ -0,0 +1,254 @@ +[[semantic-text-hybrid-search]] +=== Tutorial: hybrid search with `semantic_text` +++++ +Hybrid search with `semantic_text` +++++ + +This tutorial demonstrates how to perform hybrid search, combining semantic search with traditional full-text search. + +In hybrid search, semantic search retrieves results based on the meaning of the text, while full-text search focuses on exact word matches. By combining both methods, hybrid search delivers more relevant results, particularly in cases where relying on a single approach may not be sufficient. + +The recommended way to use hybrid search in the {stack} is following the `semantic_text` workflow. This tutorial uses the <> for demonstration, but you can use any service and its supported models offered by the {infer-cap} API. + +[discrete] +[[semantic-text-hybrid-infer-endpoint]] +==== Create the {infer} endpoint + +Create an inference endpoint by using the <>: + +[source,console] +------------------------------------------------------------ +PUT _inference/sparse_embedding/my-elser-endpoint <1> +{ + "service": "elser", <2> + "service_settings": { + "adaptive_allocations": { <3> + "enabled": true, + "min_number_of_allocations": 3, + "max_number_of_allocations": 10 + }, + "num_threads": 1 + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The task type is `sparse_embedding` in the path as the `elser` service will +be used and ELSER creates sparse vectors. The `inference_id` is +`my-elser-endpoint`. +<2> The `elser` service is used in this example. +<3> This setting enables and configures adaptive allocations. +Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process. + +[NOTE] +==== +You might see a 502 bad gateway error in the response when using the {kib} Console. +This error usually just reflects a timeout, while the model downloads in the background. +You can check the download progress in the {ml-app} UI. +==== + +[discrete] +[[hybrid-search-create-index-mapping]] +==== Create an index mapping for hybrid search + +The destination index will contain both the embeddings for semantic search and the original text field for full-text search. This structure enables the combination of semantic search and full-text search. + +[source,console] +------------------------------------------------------------ +PUT semantic-embeddings +{ + "mappings": { + "properties": { + "semantic_text": { <1> + "type": "semantic_text", + "inference_id": "my-elser-endpoint" <2> + }, + "content": { <3> + "type": "text", + "copy_to": "semantic_text" <4> + } + } + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The name of the field to contain the generated embeddings for semantic search. +<2> The identifier of the inference endpoint that generates the embeddings based on the input text. +<3> The name of the field to contain the original text for lexical search. +<4> The textual data stored in the `content` field will be copied to `semantic_text` and processed by the {infer} endpoint. + +[NOTE] +==== +If you want to run a search on indices that were populated by web crawlers or connectors, you have to +<> for these indices to +include the `semantic_text` field. Once the mapping is updated, you'll need to run a full web crawl or a full connector sync. This ensures that all existing +documents are reprocessed and updated with the new semantic embeddings, enabling hybrid search on the updated data. +==== + +[discrete] +[[semantic-text-hybrid-load-data]] +==== Load data + +In this step, you load the data that you later use to create embeddings from. + +Use the `msmarco-passagetest2019-top1000` data set, which is a subset of the MS MARCO Passage Ranking data set. It consists of 200 queries, each accompanied by a list of relevant text passages. All unique passages, along with their IDs, have been extracted from that data set and compiled into a https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file]. + +Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] in the {ml-app} UI. After your data is analyzed, click **Override settings**. Under **Edit field names**, assign `id` to the first column and `content` to the second. Click **Apply**, then **Import**. Name the index `test-data`, and click **Import**. After the upload is complete, you will see an index named `test-data` with 182,469 documents. + +[discrete] +[[hybrid-search-reindex-data]] +==== Reindex the data for hybrid search + +Reindex the data from the `test-data` index into the `semantic-embeddings` index. +The data in the `content` field of the source index is copied into the `content` field of the destination index. +The `copy_to` parameter set in the index mapping creation ensures that the content is copied into the `semantic_text` field. The data is processed by the {infer} endpoint at ingest time to generate embeddings. + +[NOTE] +==== +This step uses the reindex API to simulate data ingestion. If you are working with data that has already been indexed, +rather than using the `test-data` set, reindexing is still required to ensure that the data is processed by the {infer} endpoint +and the necessary embeddings are generated. +==== + +[source,console] +------------------------------------------------------------ +POST _reindex?wait_for_completion=false +{ + "source": { + "index": "test-data", + "size": 10 <1> + }, + "dest": { + "index": "semantic-embeddings" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The default batch size for reindexing is 1000. Reducing size to a smaller +number makes the update of the reindexing process quicker which enables you to +follow the progress closely and detect errors early. + +The call returns a task ID to monitor the progress: + +[source,console] +------------------------------------------------------------ +GET _tasks/ +------------------------------------------------------------ +// TEST[skip:TBD] + +Reindexing large datasets can take a long time. You can test this workflow using only a subset of the dataset. + +To cancel the reindexing process and generate embeddings for the subset that was reindexed: + +[source,console] +------------------------------------------------------------ +POST _tasks//_cancel +------------------------------------------------------------ +// TEST[skip:TBD] + +[discrete] +[[hybrid-search-perform-search]] +==== Perform hybrid search + +After reindexing the data into the `semantic-embeddings` index, you can perform hybrid search by using <>. RRF is a technique that merges the rankings from both semantic and lexical queries, giving more weight to results that rank high in either search. This ensures that the final results are balanced and relevant. + +[source,console] +------------------------------------------------------------ +GET semantic-embeddings/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { <1> + "query": { + "match": { + "content": "How to avoid muscle soreness while running?" <2> + } + } + } + }, + { + "standard": { <3> + "query": { + "semantic": { + "field": "semantic_text", <4> + "query": "How to avoid muscle soreness while running?" + } + } + } + } + ] + } + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The first `standard` retriever represents the traditional lexical search. +<2> Lexical search is performed on the `content` field using the specified phrase. +<3> The second `standard` retriever refers to the semantic search. +<4> The `semantic_text` field is used to perform the semantic search. + + +After performing the hybrid search, the query will return the top 10 documents that match both semantic and lexical search criteria. The results include detailed information about each document: + +[source,console-result] +------------------------------------------------------------ +{ + "took": 107, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 473, + "relation": "eq" + }, + "max_score": null, + "hits": [ + { + "_index": "semantic-embeddings", + "_id": "wv65epIBEMBRnhfTsOFM", + "_score": 0.032786883, + "_rank": 1, + "_source": { + "semantic_text": { + "inference": { + "inference_id": "my-elser-endpoint", + "model_settings": { + "task_type": "sparse_embedding" + }, + "chunks": [ + { + "text": "What so many out there do not realize is the importance of what you do after you work out. You may have done the majority of the work, but how you treat your body in the minutes and hours after you exercise has a direct effect on muscle soreness, muscle strength and growth, and staying hydrated. Cool Down. After your last exercise, your workout is not over. The first thing you need to do is cool down. Even if running was all that you did, you still should do light cardio for a few minutes. This brings your heart rate down at a slow and steady pace, which helps you avoid feeling sick after a workout.", + "embeddings": { + "exercise": 1.571044, + "after": 1.3603843, + "sick": 1.3281639, + "cool": 1.3227621, + "muscle": 1.2645415, + "sore": 1.2561599, + "cooling": 1.2335974, + "running": 1.1750668, + "hours": 1.1104802, + "out": 1.0991782, + "##io": 1.0794281, + "last": 1.0474665, + (...) + } + } + ] + } + }, + "id": 8408852, + "content": "What so many out there do not realize is the importance of (...)" + } + } + ] + } +} +------------------------------------------------------------ +// NOTCONSOLE diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java index aa869b1af4f5e..6f0b473b5ba1f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java @@ -40,6 +40,7 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.function.Function; +import java.util.function.LongFunction; /** * A utility to build XContent (ie json). @@ -107,13 +108,15 @@ public static XContentBuilder builder(XContentType xContentType, Set inc private static final Map, Writer> WRITERS; private static final Map, HumanReadableTransformer> HUMAN_READABLE_TRANSFORMERS; private static final Map, Function> DATE_TRANSFORMERS; + private static final LongFunction UNIX_EPOCH_MILLIS_FORMATTER; + static { Map, Writer> writers = new HashMap<>(); writers.put(Boolean.class, (b, v) -> b.value((Boolean) v)); writers.put(boolean[].class, (b, v) -> b.values((boolean[]) v)); writers.put(Byte.class, (b, v) -> b.value((Byte) v)); writers.put(byte[].class, (b, v) -> b.value((byte[]) v)); - writers.put(Date.class, XContentBuilder::timeValue); + writers.put(Date.class, XContentBuilder::timestampValue); writers.put(Double.class, (b, v) -> b.value((Double) v)); writers.put(double[].class, (b, v) -> b.values((double[]) v)); writers.put(Float.class, (b, v) -> b.value((Float) v)); @@ -129,8 +132,8 @@ public static XContentBuilder builder(XContentType xContentType, Set inc writers.put(Locale.class, (b, v) -> b.value(v.toString())); writers.put(Class.class, (b, v) -> b.value(v.toString())); writers.put(ZonedDateTime.class, (b, v) -> b.value(v.toString())); - writers.put(Calendar.class, XContentBuilder::timeValue); - writers.put(GregorianCalendar.class, XContentBuilder::timeValue); + writers.put(Calendar.class, XContentBuilder::timestampValue); + writers.put(GregorianCalendar.class, XContentBuilder::timestampValue); writers.put(BigInteger.class, (b, v) -> b.value((BigInteger) v)); writers.put(BigDecimal.class, (b, v) -> b.value((BigDecimal) v)); @@ -140,6 +143,8 @@ public static XContentBuilder builder(XContentType xContentType, Set inc // treat strings as already converted dateTransformers.put(String.class, Function.identity()); + LongFunction unixEpochMillisFormatter = Long::toString; + // Load pluggable extensions for (XContentBuilderExtension service : ServiceLoader.load(XContentBuilderExtension.class)) { Map, Writer> addlWriters = service.getXContentWriters(); @@ -157,11 +162,14 @@ public static XContentBuilder builder(XContentType xContentType, Set inc writers.putAll(addlWriters); humanReadableTransformer.putAll(addlTransformers); dateTransformers.putAll(addlDateTransformers); + + unixEpochMillisFormatter = service::formatUnixEpochMillis; } WRITERS = Map.copyOf(writers); HUMAN_READABLE_TRANSFORMERS = Map.copyOf(humanReadableTransformer); DATE_TRANSFORMERS = Map.copyOf(dateTransformers); + UNIX_EPOCH_MILLIS_FORMATTER = unixEpochMillisFormatter; } @FunctionalInterface @@ -797,52 +805,53 @@ public XContentBuilder utf8Value(byte[] bytes, int offset, int length) throws IO } //////////////////////////////////////////////////////////////////////////// - // Date + // Timestamps ////////////////////////////////// /** - * Write a time-based field and value, if the passed timeValue is null a - * null value is written, otherwise a date transformers lookup is performed. - - * @throws IllegalArgumentException if there is no transformers for the type of object + * Write a field with a timestamp value: if the passed timestamp is null then writes null, otherwise looks up the date transformer + * for the type of {@code timestamp} and uses it to format the value. + * + * @throws IllegalArgumentException if there is no transformer for the given value type */ - public XContentBuilder timeField(String name, Object timeValue) throws IOException { - return field(name).timeValue(timeValue); + public XContentBuilder timestampField(String name, Object timestamp) throws IOException { + return field(name).timestampValue(timestamp); } /** - * If the {@code humanReadable} flag is set, writes both a formatted and - * unformatted version of the time value using the date transformer for the - * {@link Long} class. + * Writes a field containing the raw number of milliseconds since the unix epoch, and also if the {@code humanReadable} flag is set, + * writes a formatted representation of this value using the UNIX_EPOCH_MILLIS_FORMATTER. */ - public XContentBuilder timeField(String name, String readableName, long value) throws IOException { - assert name.equals(readableName) == false : "expected raw and readable field names to differ, but they were both: " + name; + public XContentBuilder timestampFieldsFromUnixEpochMillis(String rawFieldName, String humanReadableFieldName, long unixEpochMillis) + throws IOException { + assert rawFieldName.equals(humanReadableFieldName) == false + : "expected raw and readable field names to differ, but they were both: " + rawFieldName; if (humanReadable) { - Function longTransformer = DATE_TRANSFORMERS.get(Long.class); - if (longTransformer == null) { - throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type Long"); - } - field(readableName).value(longTransformer.apply(value)); + field(humanReadableFieldName, UNIX_EPOCH_MILLIS_FORMATTER.apply(unixEpochMillis)); } - field(name, value); + field(rawFieldName, unixEpochMillis); return this; } /** - * Write a time-based value, if the value is null a null value is written, - * otherwise a date transformers lookup is performed. - - * @throws IllegalArgumentException if there is no transformers for the type of object + * Write a timestamp value: if the passed timestamp is null then writes null, otherwise looks up the date transformer for the type of + * {@code timestamp} and uses it to format the value. + * + * @throws IllegalArgumentException if there is no transformer for the given value type */ - public XContentBuilder timeValue(Object timeValue) throws IOException { - if (timeValue == null) { + public XContentBuilder timestampValue(Object timestamp) throws IOException { + if (timestamp == null) { return nullValue(); } else { - Function transformer = DATE_TRANSFORMERS.get(timeValue.getClass()); + Function transformer = DATE_TRANSFORMERS.get(timestamp.getClass()); if (transformer == null) { - throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type " + timeValue.getClass()); + final var exception = new IllegalArgumentException( + "cannot write timestamp value xcontent for value of unknown type " + timestamp.getClass() + ); + assert false : exception; + throw exception; } - return value(transformer.apply(timeValue)); + return value(transformer.apply(timestamp)); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java index 1e48667079cfc..4e3b442e7d473 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java @@ -68,4 +68,9 @@ public interface XContentBuilderExtension { * */ Map, Function> getDateTransformers(); + + /** + * Used to format a {@code long} representing the number of milliseconds since the Unix Epoch. + */ + String formatUnixEpochMillis(long unixEpochMillis); } diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java index 3e66bf0edf394..e911bf1a41198 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java @@ -65,17 +65,17 @@ protected Collection> nodePlugins() { } private static IndexRequestBuilder indexDoc(String idx, ZonedDateTime date, int value) throws Exception { - return prepareIndex(idx).setSource(jsonBuilder().startObject().timeField("date", date).field("value", value).endObject()); + return prepareIndex(idx).setSource(jsonBuilder().startObject().timestampField("date", date).field("value", value).endObject()); } private IndexRequestBuilder indexDoc(int month, int day, int value) throws Exception { return prepareIndex("idx").setSource( jsonBuilder().startObject() .field("value", value) - .timeField("date", date(month, day)) + .timestampField("date", date(month, day)) .startArray("dates") - .timeValue(date(month, day)) - .timeValue(date(month + 1, day + 1)) + .timestampValue(date(month, day)) + .timestampValue(date(month + 1, day + 1)) .endArray() .endObject() ); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java new file mode 100644 index 0000000000000..980cc32a12c68 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +/** + * This should be a yaml test, but in order to write one we would need to expose the new APIs in the rest-api-spec. + * We do not want to do that until the feature flag is removed. For this reason, we temporarily, test the new APIs here. + * Please convert this to a yaml test when the feature flag is removed. + */ +public class DataStreamOptionsIT extends DisabledSecurityDataStreamTestCase { + + private static final String DATA_STREAM_NAME = "failure-data-stream"; + + @SuppressWarnings("unchecked") + @Before + public void setup() throws IOException { + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/ds-template"); + putComposableIndexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["failure-data-stream"], + "template": { + "settings": { + "number_of_replicas": 0 + } + }, + "data_stream": { + "failure_store": true + } + } + """); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME))); + // Initialize the failure store. + assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store"))); + ensureGreen(DATA_STREAM_NAME); + + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + List backingIndices = getIndices(dataStream); + assertThat(backingIndices.size(), is(1)); + List failureStore = getFailureStore(dataStream); + assertThat(failureStore.size(), is(1)); + } + + public void testEnableDisableFailureStore() throws IOException { + { + assertAcknowledged(client().performRequest(new Request("DELETE", "/_data_stream/" + DATA_STREAM_NAME + "/_options"))); + assertFailureStore(false, 1); + assertDataStreamOptions(null); + } + { + Request enableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options"); + enableRequest.setJsonEntity(""" + { + "failure_store": { + "enabled": true + } + }"""); + assertAcknowledged(client().performRequest(enableRequest)); + assertFailureStore(true, 1); + assertDataStreamOptions(true); + } + + { + Request disableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options"); + disableRequest.setJsonEntity(""" + { + "failure_store": { + "enabled": false + } + }"""); + assertAcknowledged(client().performRequest(disableRequest)); + assertFailureStore(false, 1); + assertDataStreamOptions(false); + } + } + + @SuppressWarnings("unchecked") + private void assertFailureStore(boolean failureStoreEnabled, int failureStoreSize) throws IOException { + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + assertThat(dataStream.containsKey("failure_store"), is(true)); + // Ensure the failure store is set to the provided value + assertThat(((Map) dataStream.get("failure_store")).get("enabled"), equalTo(failureStoreEnabled)); + // And the failure indices preserved + List failureStore = getFailureStore(dataStream); + assertThat(failureStore.size(), is(failureStoreSize)); + } + + @SuppressWarnings("unchecked") + private void assertDataStreamOptions(Boolean failureStoreEnabled) throws IOException { + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME + "/_options")); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + Map> options = (Map>) dataStream.get("options"); + if (failureStoreEnabled == null) { + assertThat(options, nullValue()); + } else { + assertThat(options.containsKey("failure_store"), is(true)); + assertThat(options.get("failure_store").get("enabled"), equalTo(failureStoreEnabled)); + } + } + + @SuppressWarnings("unchecked") + private List getFailureStore(Map response) { + var failureStore = (Map) response.get("failure_store"); + return getIndices(failureStore); + + } + + @SuppressWarnings("unchecked") + private List getIndices(Map response) { + List> indices = (List>) response.get("indices"); + return indices.stream().map(index -> index.get("index_name")).toList(); + } +} diff --git a/modules/data-streams/src/main/java/module-info.java b/modules/data-streams/src/main/java/module-info.java index 16229f9eb2394..2d49029c1023c 100644 --- a/modules/data-streams/src/main/java/module-info.java +++ b/modules/data-streams/src/main/java/module-info.java @@ -17,6 +17,7 @@ exports org.elasticsearch.datastreams.action to org.elasticsearch.server; exports org.elasticsearch.datastreams.lifecycle.action to org.elasticsearch.server; exports org.elasticsearch.datastreams.lifecycle; + exports org.elasticsearch.datastreams.options.action to org.elasticsearch.server; provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.datastreams.DataStreamFeatures; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index a3d0347c3d192..d6a0fd86265e5 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -59,7 +59,7 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider { public Settings getAdditionalIndexSettings( String indexName, @Nullable String dataStreamName, - boolean isTimeSeries, + @Nullable IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, @@ -70,15 +70,16 @@ public Settings getAdditionalIndexSettings( // First backing index is created and then data stream is rolled over (in a single cluster state update). // So at this point we can't check index_mode==time_series, // so checking that index_mode==null|standard and templateIndexMode == TIME_SERIES + boolean isMigratingToTimeSeries = templateIndexMode == IndexMode.TIME_SERIES; boolean migrating = dataStream != null && (dataStream.getIndexMode() == null || dataStream.getIndexMode() == IndexMode.STANDARD) - && isTimeSeries; + && isMigratingToTimeSeries; IndexMode indexMode; if (migrating) { indexMode = IndexMode.TIME_SERIES; } else if (dataStream != null) { - indexMode = isTimeSeries ? dataStream.getIndexMode() : null; - } else if (isTimeSeries) { + indexMode = isMigratingToTimeSeries ? dataStream.getIndexMode() : null; + } else if (isMigratingToTimeSeries) { indexMode = IndexMode.TIME_SERIES; } else { indexMode = null; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 1a6465a251021..cb7445705537a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -56,6 +57,15 @@ import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamLifecycleAction; +import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportDeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportGetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportPutDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestDeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestGetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestPutDataStreamOptionsAction; import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction; import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction; import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction; @@ -229,6 +239,11 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(DeleteDataStreamLifecycleAction.INSTANCE, TransportDeleteDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(ExplainDataStreamLifecycleAction.INSTANCE, TransportExplainDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(GetDataStreamLifecycleStatsAction.INSTANCE, TransportGetDataStreamLifecycleStatsAction.class)); + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class)); + } return actions; } @@ -261,6 +276,11 @@ public List getRestHandlers( handlers.add(new RestDeleteDataStreamLifecycleAction()); handlers.add(new RestExplainDataStreamLifecycleAction()); handlers.add(new RestDataStreamLifecycleStatsAction()); + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + handlers.add(new RestGetDataStreamOptionsAction()); + handlers.add(new RestPutDataStreamOptionsAction()); + handlers.add(new RestDeleteDataStreamOptionsAction()); + } return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..98a29dd636ddf --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * Removes the data stream options configuration from the requested data streams. + */ +public class DeleteDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/delete"); + + private DeleteDataStreamOptionsAction() {/* no instances */} + + public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readOptionalStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalStringArray(names); + indicesOptions.writeIndicesOptions(out); + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + } + + public String[] getNames() { + return names; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) && Objects.equals(indicesOptions, request.indicesOptions); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..c1354da1129ca --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * This action retrieves the data stream options from every data stream. Currently, data stream options only support + * failure store. + */ +public class GetDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/get"); + + private GetDataStreamOptionsAction() {/* no instances */} + + public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + private boolean includeDefaults = false; + + public Request(TimeValue masterNodeTimeout, String[] names) { + super(masterNodeTimeout); + this.names = names; + } + + public Request(TimeValue masterNodeTimeout, String[] names, boolean includeDefaults) { + super(masterNodeTimeout); + this.names = names; + this.includeDefaults = includeDefaults; + } + + public String[] getNames() { + return names; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readOptionalStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + this.includeDefaults = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalStringArray(names); + indicesOptions.writeIndicesOptions(out); + out.writeBoolean(includeDefaults); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) + && indicesOptions.equals(request.indicesOptions) + && includeDefaults == request.includeDefaults; + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, includeDefaults); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public boolean includeDefaults() { + return includeDefaults; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } + + public Request includeDefaults(boolean includeDefaults) { + this.includeDefaults = includeDefaults; + return this; + } + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams"); + + public record DataStreamEntry(String dataStreamName, DataStreamOptions dataStreamOptions) implements Writeable, ToXContentObject { + + public static final ParseField NAME_FIELD = new ParseField("name"); + public static final ParseField OPTIONS_FIELD = new ParseField("options"); + + DataStreamEntry(StreamInput in) throws IOException { + this(in.readString(), in.readOptionalWriteable(DataStreamOptions::read)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStreamName); + out.writeOptionalWriteable(dataStreamOptions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD.getPreferredName(), dataStreamName); + if (dataStreamOptions != null && dataStreamOptions.isEmpty() == false) { + builder.field(OPTIONS_FIELD.getPreferredName(), dataStreamOptions); + } + builder.endObject(); + return builder; + } + } + + private final List dataStreams; + + public Response(List dataStreams) { + this.dataStreams = dataStreams; + } + + public Response(StreamInput in) throws IOException { + this(in.readCollectionAsList(DataStreamEntry::new)); + } + + public List getDataStreams() { + return dataStreams; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(dataStreams); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params outerParams) { + return Iterators.concat(Iterators.single((builder, params) -> { + builder.startObject(); + builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); + return builder; + }), + Iterators.map(dataStreams.iterator(), entry -> (builder, params) -> entry.toXContent(builder, outerParams)), + Iterators.single((builder, params) -> { + builder.endArray(); + builder.endObject(); + return builder; + }) + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return dataStreams.equals(response.dataStreams); + } + + @Override + public int hashCode() { + return Objects.hash(dataStreams); + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..d055a6972312a --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.DataStreamFailureStore; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Sets the data stream options that was provided in the request to the requested data streams. + */ +public class PutDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/put"); + + private PutDataStreamOptionsAction() {/* no instances */} + + public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + + public interface Factory { + Request create(@Nullable DataStreamFailureStore dataStreamFailureStore); + } + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "put_data_stream_options_request", + false, + (args, factory) -> factory.create((DataStreamFailureStore) args[0]) + ); + + static { + PARSER.declareObjectOrNull( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DataStreamFailureStore.PARSER.parse(p, null), + null, + new ParseField("failure_store") + ); + } + + public static Request parseRequest(XContentParser parser, Factory factory) { + return PARSER.apply(parser, factory); + } + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + private final DataStreamOptions options; + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + options = DataStreamOptions.read(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(names); + indicesOptions.writeIndicesOptions(out); + out.writeWriteable(options); + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamOptions options) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + this.options = options; + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable DataStreamFailureStore failureStore) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + this.options = new DataStreamOptions(failureStore); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (options.failureStore() == null) { + validationException = addValidationError("At least one option needs to be provided", validationException); + } + return validationException; + } + + public String[] getNames() { + return names; + } + + public DataStreamOptions getOptions() { + return options; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) + && Objects.equals(indicesOptions, request.indicesOptions) + && options.equals(request.options); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, options); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public IndicesRequest indices(String... names) { + this.names = names; + return this; + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..ead23ed78222b --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.List; + +/** + * Transport action that resolves the data stream names from the request and removes any configured data stream options from them. + */ +public class TransportDeleteDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction { + + private final MetadataDataStreamsService metadataDataStreamsService; + private final SystemIndices systemIndices; + + @Inject + public TransportDeleteDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataDataStreamsService metadataDataStreamsService, + SystemIndices systemIndices + ) { + super( + DeleteDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + DeleteDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + DeleteDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List dataStreamNames = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + for (String name : dataStreamNames) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + metadataDataStreamsService.removeDataStreamOptions(dataStreamNames, request.ackTimeout(), request.masterNodeTimeout(), listener); + } + + @Override + protected ClusterBlockException checkBlock(DeleteDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..b032b35c943c0 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Collects the data streams from the cluster state and then returns for each data stream its name and its + * data stream options. Currently, data stream options include only the failure store configuration. + */ +public class TransportGetDataStreamOptionsAction extends TransportMasterNodeReadAction< + GetDataStreamOptionsAction.Request, + GetDataStreamOptionsAction.Response> { + + private final SystemIndices systemIndices; + + @Inject + public TransportGetDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices + ) { + super( + GetDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + GetDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + GetDataStreamOptionsAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + GetDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List requestedDataStreams = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + Map dataStreams = state.metadata().dataStreams(); + for (String name : requestedDataStreams) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + listener.onResponse( + new GetDataStreamOptionsAction.Response( + requestedDataStreams.stream() + .map(dataStreams::get) + .filter(Objects::nonNull) + .map( + dataStream -> new GetDataStreamOptionsAction.Response.DataStreamEntry( + dataStream.getName(), + dataStream.getDataStreamOptions() + ) + ) + .sorted(Comparator.comparing(GetDataStreamOptionsAction.Response.DataStreamEntry::dataStreamName)) + .toList() + ) + ); + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..b1386232c44f9 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.List; + +/** + * Transport action that resolves the data stream names from the request and sets the data stream lifecycle provided in the request. + */ +public class TransportPutDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction { + + private final MetadataDataStreamsService metadataDataStreamsService; + private final SystemIndices systemIndices; + + @Inject + public TransportPutDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataDataStreamsService metadataDataStreamsService, + SystemIndices systemIndices + ) { + super( + PutDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + PutDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + PutDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List dataStreamNames = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + for (String name : dataStreamNames) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + metadataDataStreamsService.setDataStreamOptions( + dataStreamNames, + request.getOptions(), + request.ackTimeout(), + request.masterNodeTimeout(), + listener + ); + } + + @Override + protected ClusterBlockException checkBlock(PutDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..96460632ff443 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.INTERNAL) +public class RestDeleteDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "delete_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(DELETE, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + final var deleteDataOptionsRequest = new DeleteDataStreamOptionsAction.Request( + getMasterNodeTimeout(request), + request.paramAsTime("timeout", AcknowledgedRequest.DEFAULT_ACK_TIMEOUT), + Strings.splitStringByCommaToArray(request.param("name")) + ); + deleteDataOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteDataOptionsRequest.indicesOptions())); + return channel -> client.execute( + DeleteDataStreamOptionsAction.INSTANCE, + deleteDataOptionsRequest, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..6d6530efce1b9 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestGetDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + GetDataStreamOptionsAction.Request getDataStreamOptionsRequest = new GetDataStreamOptionsAction.Request( + RestUtils.getMasterNodeTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")) + ); + getDataStreamOptionsRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); + getDataStreamOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, getDataStreamOptionsRequest.indicesOptions())); + return channel -> client.execute( + GetDataStreamOptionsAction.INSTANCE, + getDataStreamOptionsRequest, + new RestRefCountedChunkedToXContentListener<>(channel) + ); + } + + @Override + public boolean allowSystemIndexAccessByDefault() { + return true; + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..9191b96b6039e --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.rest.RestUtils.getAckTimeout; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.PUBLIC) +public class RestPutDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "put_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + try (XContentParser parser = request.contentParser()) { + PutDataStreamOptionsAction.Request putOptionsRequest = PutDataStreamOptionsAction.Request.parseRequest( + parser, + (failureStore) -> new PutDataStreamOptionsAction.Request( + getMasterNodeTimeout(request), + getAckTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")), + failureStore + ) + ); + putOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, putOptionsRequest.indicesOptions())); + return channel -> client.execute(PutDataStreamOptionsAction.INSTANCE, putOptionsRequest, new RestToXContentListener<>(channel)); + } + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index d8d4a9c03933a..015752724cb5d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -78,7 +78,7 @@ public void testGetAdditionalIndexSettings() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -123,7 +123,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -193,7 +193,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -218,7 +218,7 @@ public void testGetAdditionalIndexSettingsNoMappings() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -243,7 +243,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -268,7 +268,7 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -299,7 +299,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -336,7 +336,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi () -> provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -362,7 +362,7 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - false, + null, metadata, Instant.ofEpochMilli(1L), settings, @@ -382,7 +382,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -415,7 +415,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - false, + null, metadata, Instant.ofEpochMilli(1L), settings, @@ -694,7 +694,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, diff --git a/modules/ingest-geoip/src/main/java/module-info.java b/modules/ingest-geoip/src/main/java/module-info.java index f64c30c060b08..0703d9fb449aa 100644 --- a/modules/ingest-geoip/src/main/java/module-info.java +++ b/modules/ingest-geoip/src/main/java/module-info.java @@ -18,4 +18,6 @@ exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server; exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server; + + exports org.elasticsearch.ingest.geoip to com.maxmind.db; } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java index 865d0c5a9eca0..3d2b54b04695f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java @@ -73,14 +73,14 @@ void updateDatabase(Path file, boolean update) { String databaseFileName = file.getFileName().toString(); try { if (update) { - logger.info("database file changed [{}], reload database...", file); + logger.info("database file changed [{}], reloading database...", file); DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, null); DatabaseReaderLazyLoader existing = configDatabases.put(databaseFileName, loader); if (existing != null) { existing.shutdown(); } } else { - logger.info("database file removed [{}], close database...", file); + logger.info("database file removed [{}], closing database...", file); DatabaseReaderLazyLoader existing = configDatabases.remove(databaseFileName); assert existing != null; existing.shutdown(); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 6c64cb755bb32..9508bf0346058 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -196,13 +196,19 @@ public IpDatabase get() throws IOException { } if (Assertions.ENABLED) { - // Only check whether the suffix has changed and not the entire database type. - // To sanity check whether a city db isn't overwriting with a country or asn db. - // For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different, - // by checking just the suffix this assertion doesn't fail. - String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-')); - assert loader.getDatabaseType().endsWith(expectedSuffix) - : "database type [" + loader.getDatabaseType() + "] doesn't match with expected suffix [" + expectedSuffix + "]"; + // Note that the expected suffix might be null for providers that aren't amenable to using dashes as separator for + // determining the database type. + int last = databaseType.lastIndexOf('-'); + final String expectedSuffix = last == -1 ? null : databaseType.substring(last); + + // If the entire database type matches, then that's a match. Otherwise, if there's a suffix to compare on, then + // check whether the suffix has changed (not the entire database type). + // This is to sanity check, for example, that a city db isn't overwritten with a country or asn db. + // But there are permissible overwrites that make sense, for example overwriting a geolite city db with a geoip city db + // is a valid change, but the db type is slightly different -- by checking just the suffix this assertion won't fail. + final String loaderType = loader.getDatabaseType(); + assert loaderType.equals(databaseType) || expectedSuffix == null || loaderType.endsWith(expectedSuffix) + : "database type [" + loaderType + "] doesn't match with expected suffix [" + expectedSuffix + "]"; } return loader; } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java index 3379fdff0633a..e879f0e0e3514 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java @@ -13,9 +13,16 @@ import org.elasticsearch.core.Nullable; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.function.Function; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.IPINFO_PREFIX; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoDatabase; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoLookup; +import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindDatabase; +import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindLookup; + final class IpDataLookupFactories { private IpDataLookupFactories() { @@ -26,78 +33,44 @@ interface IpDataLookupFactory { IpDataLookup create(List properties); } - private static final String CITY_DB_SUFFIX = "-City"; - private static final String COUNTRY_DB_SUFFIX = "-Country"; - private static final String ASN_DB_SUFFIX = "-ASN"; - private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; - private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; - private static final String DOMAIN_DB_SUFFIX = "-Domain"; - private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; - private static final String ISP_DB_SUFFIX = "-ISP"; - - @Nullable - private static Database getMaxmindDatabase(final String databaseType) { - if (databaseType.endsWith(CITY_DB_SUFFIX)) { - return Database.City; - } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { - return Database.Country; - } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { - return Database.Asn; - } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) { - return Database.AnonymousIp; - } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) { - return Database.ConnectionType; - } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) { - return Database.Domain; - } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) { - return Database.Enterprise; - } else if (databaseType.endsWith(ISP_DB_SUFFIX)) { - return Database.Isp; - } else { - return null; // no match was found - } - } - /** * Parses the passed-in databaseType and return the Database instance that is * associated with that databaseType. * * @param databaseType the database type String from the metadata of the database file - * @return the Database instance that is associated with the databaseType + * @return the Database instance that is associated with the databaseType (or null) */ @Nullable static Database getDatabase(final String databaseType) { Database database = null; if (Strings.hasText(databaseType)) { - database = getMaxmindDatabase(databaseType); + final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT); + if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) { + database = getIpinfoDatabase(databaseTypeLowerCase); // all lower case! + } else { + // for historical reasons, fall back to assuming maxmind-like type parsing + database = getMaxmindDatabase(databaseType); + } } return database; } - @Nullable - static Function, IpDataLookup> getMaxmindLookup(final Database database) { - return switch (database) { - case City -> MaxmindIpDataLookups.City::new; - case Country -> MaxmindIpDataLookups.Country::new; - case Asn -> MaxmindIpDataLookups.Asn::new; - case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new; - case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new; - case Domain -> MaxmindIpDataLookups.Domain::new; - case Enterprise -> MaxmindIpDataLookups.Enterprise::new; - case Isp -> MaxmindIpDataLookups.Isp::new; - default -> null; - }; - } - static IpDataLookupFactory get(final String databaseType, final String databaseFile) { final Database database = getDatabase(databaseType); if (database == null) { throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); } - final Function, IpDataLookup> factoryMethod = getMaxmindLookup(database); + final Function, IpDataLookup> factoryMethod; + final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT); + if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) { + factoryMethod = getIpinfoLookup(database); + } else { + // for historical reasons, fall back to assuming maxmind-like types + factoryMethod = getMaxmindLookup(database); + } if (factoryMethod == null) { throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java index efc6734b3bd93..19a98fb1b5746 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java @@ -23,10 +23,14 @@ import java.io.IOException; import java.net.InetAddress; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; /** * A collection of {@link IpDataLookup} implementations for IPinfo databases @@ -43,6 +47,81 @@ private IpinfoIpDataLookups() { // prefix dispatch and checks case-insensitive, so that works out nicely static final String IPINFO_PREFIX = "ipinfo"; + private static final Set IPINFO_TYPE_STOP_WORDS = Set.of( + "ipinfo", + "extended", + "free", + "generic", + "ip", + "sample", + "standard", + "mmdb" + ); + + /** + * Cleans up the database_type String from an ipinfo database by splitting on punctuation, removing stop words, and then joining + * with an underscore. + *

+ * e.g. "ipinfo free_foo_sample.mmdb" -> "foo" + * + * @param type the database_type from an ipinfo database + * @return a cleaned up database_type string + */ + // n.b. this is just based on observation of the types from a survey of such databases -- it's like browser user agent sniffing, + // there aren't necessarily any amazing guarantees about this behavior + static String ipinfoTypeCleanup(String type) { + List parts = Arrays.asList(type.split("[ _.]")); + return parts.stream().filter((s) -> IPINFO_TYPE_STOP_WORDS.contains(s) == false).collect(Collectors.joining("_")); + } + + @Nullable + static Database getIpinfoDatabase(final String databaseType) { + // for ipinfo the database selection is more along the lines of user-agent sniffing than + // string-based dispatch. the specific database_type strings could change in the future, + // hence the somewhat loose nature of this checking. + + final String cleanedType = ipinfoTypeCleanup(databaseType); + + // early detection on any of the 'extended' types + if (databaseType.contains("extended")) { + // which are not currently supported + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + + // early detection on 'country_asn' so the 'country' and 'asn' checks don't get faked out + if (cleanedType.contains("country_asn")) { + // but it's not currently supported + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + + if (cleanedType.contains("asn")) { + return Database.AsnV2; + } else if (cleanedType.contains("country")) { + return Database.CountryV2; + } else if (cleanedType.contains("location")) { // note: catches 'location' and 'geolocation' ;) + return Database.CityV2; + } else if (cleanedType.contains("privacy")) { + return Database.PrivacyDetection; + } else { + // no match was found + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + } + + @Nullable + static Function, IpDataLookup> getIpinfoLookup(final Database database) { + return switch (database) { + case Database.AsnV2 -> IpinfoIpDataLookups.Asn::new; + case Database.CountryV2 -> IpinfoIpDataLookups.Country::new; + case Database.CityV2 -> IpinfoIpDataLookups.Geolocation::new; + case Database.PrivacyDetection -> IpinfoIpDataLookups.PrivacyDetection::new; + default -> null; + }; + } + /** * Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible). * @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java index 4297413073e52..8bc74c0e4aac4 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java @@ -26,6 +26,8 @@ import com.maxmind.geoip2.record.Postal; import com.maxmind.geoip2.record.Subdivision; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.Nullable; @@ -37,6 +39,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.Function; /** * A collection of {@link IpDataLookup} implementations for MaxMind databases @@ -47,11 +50,63 @@ private MaxmindIpDataLookups() { // utility class } + private static final Logger logger = LogManager.getLogger(MaxmindIpDataLookups.class); + // the actual prefixes from the metadata are cased like the literal strings, but // prefix dispatch and checks case-insensitive, so the actual constants are lowercase static final String GEOIP2_PREFIX = "GeoIP2".toLowerCase(Locale.ROOT); static final String GEOLITE2_PREFIX = "GeoLite2".toLowerCase(Locale.ROOT); + // note: the secondary dispatch on suffix happens to be case sensitive + private static final String CITY_DB_SUFFIX = "-City"; + private static final String COUNTRY_DB_SUFFIX = "-Country"; + private static final String ASN_DB_SUFFIX = "-ASN"; + private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; + private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; + private static final String DOMAIN_DB_SUFFIX = "-Domain"; + private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; + private static final String ISP_DB_SUFFIX = "-ISP"; + + @Nullable + static Database getMaxmindDatabase(final String databaseType) { + if (databaseType.endsWith(CITY_DB_SUFFIX)) { + return Database.City; + } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { + return Database.Country; + } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { + return Database.Asn; + } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) { + return Database.AnonymousIp; + } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) { + return Database.ConnectionType; + } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) { + return Database.Domain; + } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) { + return Database.Enterprise; + } else if (databaseType.endsWith(ISP_DB_SUFFIX)) { + return Database.Isp; + } else { + // no match was found + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + } + + @Nullable + static Function, IpDataLookup> getMaxmindLookup(final Database database) { + return switch (database) { + case City -> MaxmindIpDataLookups.City::new; + case Country -> MaxmindIpDataLookups.Country::new; + case Asn -> MaxmindIpDataLookups.Asn::new; + case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new; + case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new; + case Domain -> MaxmindIpDataLookups.Domain::new; + case Enterprise -> MaxmindIpDataLookups.Enterprise::new; + case Isp -> MaxmindIpDataLookups.Isp::new; + default -> null; + }; + } + static class AnonymousIp extends AbstractBase { AnonymousIp(final Set properties) { super( diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java index 82888fa39c857..fcfd8e51aabb5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java @@ -66,7 +66,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws // (we'll be a in a json map where the id is the key) builder.startObject(); builder.field(VERSION.getPreferredName(), version); - builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate); + builder.timestampFieldsFromUnixEpochMillis(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate); builder.field(DATABASE.getPreferredName(), database); builder.endObject(); return builder; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java index 89bc3d1ce5d37..1970883e91b3e 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java @@ -110,7 +110,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("id", database.id()); // serialize including the id -- this is get response serialization builder.field(VERSION.getPreferredName(), item.version()); - builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), item.modifiedDate()); + builder.timestampFieldsFromUnixEpochMillis( + MODIFIED_DATE_MILLIS.getPreferredName(), + MODIFIED_DATE.getPreferredName(), + item.modifiedDate() + ); builder.field(DATABASE.getPreferredName(), database); builder.endObject(); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index e96bdbd6314b2..640480ed277c5 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -27,6 +27,7 @@ import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -37,10 +38,6 @@ public class GeoIpProcessorTests extends ESTestCase { - private static IpDataLookup ipDataLookupAll(final Database database) { - return IpDataLookupFactories.getMaxmindLookup(database).apply(database.properties()); - } - // a temporary directory that mmdb files can be copied to and read from private Path tmpDir; @@ -54,6 +51,66 @@ public void cleanup() throws IOException { IOUtils.rm(tmpDir); } + public void testMaxmindCity() throws Exception { + String ip = "2602:306:33d3:8000::3257:9652"; + GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, // n.b. this is a "geoip" processor + randomAlphaOfLength(10), + null, + "source_field", + loader("GeoLite2-City.mmdb"), + () -> true, + "target_field", + getMaxmindCityLookup(), + false, + false, + "filename" + ); + + Map document = new HashMap<>(); + document.put("source_field", ip); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + + assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); + @SuppressWarnings("unchecked") + Map data = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.get("ip"), equalTo(ip)); + assertThat(data.get("city_name"), equalTo("Homestead")); + // see MaxmindIpDataLookupsTests for more tests of the data lookup behavior + } + + public void testIpinfoGeolocation() throws Exception { + String ip = "13.107.39.238"; + GeoIpProcessor processor = new GeoIpProcessor( + IP_LOCATION_TYPE, // n.b. this is an "ip_location" processor + randomAlphaOfLength(10), + null, + "source_field", + loader("ipinfo/ip_geolocation_sample.mmdb"), + () -> true, + "target_field", + getIpinfoGeolocationLookup(), + false, + false, + "filename" + ); + + Map document = new HashMap<>(); + document.put("source_field", ip); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + + assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); + @SuppressWarnings("unchecked") + Map data = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.get("ip"), equalTo(ip)); + assertThat(data.get("city_name"), equalTo("Des Moines")); + // see IpinfoIpDataLookupsTests for more tests of the data lookup behavior + } + public void testNullValueWithIgnoreMissing() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( GEOIP_TYPE, @@ -63,7 +120,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), true, false, "filename" @@ -86,7 +143,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), true, false, "filename" @@ -106,7 +163,7 @@ public void testNullWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "filename" @@ -129,7 +186,7 @@ public void testNonExistentWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "filename" @@ -149,7 +206,7 @@ public void testAddressIsNotInTheDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "filename" @@ -174,7 +231,7 @@ public void testExceptionPropagates() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "filename" @@ -196,7 +253,7 @@ public void testListAllValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "filename" @@ -208,11 +265,11 @@ public void testListAllValid() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - List> geoData = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); - assertThat(geoData.get(1).get("city_name"), equalTo("Hoensbroek")); + List> data = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.size(), equalTo(2)); + assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); + assertThat(data.get(1).get("city_name"), equalTo("Hoensbroek")); } public void testListPartiallyValid() throws Exception { @@ -224,7 +281,7 @@ public void testListPartiallyValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "filename" @@ -236,11 +293,11 @@ public void testListPartiallyValid() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - List> geoData = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); - assertThat(geoData.get(1), nullValue()); + List> data = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.size(), equalTo(2)); + assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); + assertThat(data.get(1), nullValue()); } public void testListNoMatches() throws Exception { @@ -252,7 +309,7 @@ public void testListNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "filename" @@ -272,7 +329,7 @@ public void testListDatabaseReferenceCounting() throws Exception { GeoIpProcessor processor = new GeoIpProcessor(GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", () -> { loader.preLookup(); return loader; - }, () -> true, "target_field", ipDataLookupAll(Database.City), false, false, "filename"); + }, () -> true, "target_field", getMaxmindCityLookup(), false, false, "filename"); Map document = new HashMap<>(); document.put("source_field", List.of("8.8.8.8", "82.171.64.0")); @@ -280,11 +337,11 @@ public void testListDatabaseReferenceCounting() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - List> geoData = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); - assertThat(geoData.get(1).get("city_name"), equalTo("Hoensbroek")); + List> data = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.size(), equalTo(2)); + assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); + assertThat(data.get(1).get("city_name"), equalTo("Hoensbroek")); // Check the loader's reference count and attempt to close assertThat(loader.current(), equalTo(0)); @@ -301,7 +358,7 @@ public void testListFirstOnly() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, true, "filename" @@ -313,9 +370,9 @@ public void testListFirstOnly() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); + Map data = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); } public void testListFirstOnlyNoMatches() throws Exception { @@ -327,7 +384,7 @@ public void testListFirstOnlyNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, true, "filename" @@ -350,7 +407,7 @@ public void testInvalidDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> false, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, true, "filename" @@ -374,7 +431,7 @@ public void testNoDatabase() throws Exception { () -> null, () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), false, false, "GeoLite2-City" @@ -398,7 +455,7 @@ public void testNoDatabase_ignoreMissing() throws Exception { () -> null, () -> true, "target_field", - ipDataLookupAll(Database.City), + getMaxmindCityLookup(), true, false, "GeoLite2-City" @@ -412,13 +469,24 @@ public void testNoDatabase_ignoreMissing() throws Exception { assertIngestDocument(originalIngestDocument, ingestDocument); } + private static IpDataLookup getMaxmindCityLookup() { + final var database = Database.City; + return MaxmindIpDataLookups.getMaxmindLookup(database).apply(database.properties()); + } + + private static IpDataLookup getIpinfoGeolocationLookup() { + final var database = Database.CityV2; + return IpinfoIpDataLookups.getIpinfoLookup(database).apply(database.properties()); + } + private CheckedSupplier loader(final String path) { var loader = loader(path, null); return () -> loader; } private DatabaseReaderLazyLoader loader(final String databaseName, final AtomicBoolean closed) { - Path path = tmpDir.resolve(databaseName); + int last = databaseName.lastIndexOf("/"); + final Path path = tmpDir.resolve(last == -1 ? databaseName : databaseName.substring(last + 1)); copyDatabase(databaseName, path); final GeoIpCache cache = new GeoIpCache(1000); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java index 4167170567f52..e998748efbcad 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -31,12 +31,14 @@ import static java.util.Map.entry; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.ipinfoTypeCleanup; import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn; import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseBoolean; import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseLocationDouble; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; @@ -308,6 +310,107 @@ public void testPrivacyDetectionInvariants() { } } + public void testIpinfoTypeCleanup() { + Map typesToCleanedTypes = Map.ofEntries( + // database_type strings from upstream: + // abuse.mmdb + entry("ipinfo standard_abuse_mmdb_v4.mmdb", "abuse_v4"), + // asn.mmdb + entry("ipinfo generic_asn_mmdb_v4.mmdb", "asn_v4"), + // carrier.mmdb + entry("ipinfo standard_carrier_mmdb.mmdb", "carrier"), + // location_extended_v2.mmdb + entry("ipinfo extended_location_v2.mmdb", "location_v2"), + // privacy_extended_v2.mmdb + entry("ipinfo extended_privacy_v2.mmdb", "privacy_v2"), + // standard_company.mmdb + entry("ipinfo standard_company.mmdb", "company"), + // standard_ip_hosted_domains_sample.mmdb + entry("ipinfo standard_ip_hosted_domains_sample.mmdb", "hosted_domains"), + // standard_location.mmdb + entry("ipinfo standard_location_mmdb_v4.mmdb", "location_v4"), + // standard_privacy.mmdb + entry("ipinfo standard_privacy.mmdb", "privacy"), + + // database_type strings from test files: + // ip_asn_sample.mmdb + entry("ipinfo ip_asn_sample.mmdb", "asn"), + // ip_country_asn_sample.mmdb + entry("ipinfo ip_country_asn_sample.mmdb", "country_asn"), + // ip_geolocation_sample.mmdb + entry("ipinfo ip_geolocation_sample.mmdb", "geolocation"), + // abuse_contact_sample.mmdb + entry("ipinfo abuse_contact_sample.mmdb", "abuse_contact"), + // asn_sample.mmdb + entry("ipinfo asn_sample.mmdb", "asn"), + // hosted_domains_sample.mmdb + entry("ipinfo hosted_domains_sample.mmdb", "hosted_domains"), + // ip_carrier_sample.mmdb + entry("ipinfo ip_carrier_sample.mmdb", "carrier"), + // ip_company_sample.mmdb + entry("ipinfo ip_company_sample.mmdb", "company"), + // ip_country_sample.mmdb + entry("ipinfo ip_country_sample.mmdb", "country"), + // ip_geolocation_extended_ipv4_sample.mmdb + entry("ipinfo ip_geolocation_extended_ipv4_sample.mmdb", "geolocation_ipv4"), + // ip_geolocation_extended_ipv6_sample.mmdb + entry("ipinfo ip_geolocation_extended_ipv6_sample.mmdb", "geolocation_ipv6"), + // ip_geolocation_extended_sample.mmdb + entry("ipinfo ip_geolocation_extended_sample.mmdb", "geolocation"), + // ip_rdns_domains_sample.mmdb + entry("ipinfo ip_rdns_domains_sample.mmdb", "rdns_domains"), + // ip_rdns_hostnames_sample.mmdb + entry("ipinfo ip_rdns_hostnames_sample.mmdb", "rdns_hostnames"), + // privacy_detection_extended_sample.mmdb + entry("ipinfo privacy_detection_extended_sample.mmdb", "privacy_detection"), + // privacy_detection_sample.mmdb + entry("ipinfo privacy_detection_sample.mmdb", "privacy_detection"), + + // database_type strings from downloaded (free) files: + // asn.mmdb + entry("ipinfo generic_asn_free.mmdb", "asn"), + // country.mmdb + entry("ipinfo generic_country_free.mmdb", "country"), + // country_asn.mmdb + entry("ipinfo generic_country_free_country_asn.mmdb", "country_country_asn") + ); + + for (var entry : typesToCleanedTypes.entrySet()) { + String type = entry.getKey(); + String cleanedType = entry.getValue(); + assertThat(ipinfoTypeCleanup(type), equalTo(cleanedType)); + } + } + + public void testDatabaseTypeParsing() throws IOException { + // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of + // database_type strings from these files, *and* it's also testing that we dispatch on those strings + // correctly and associated those files with the correct high-level Elasticsearch Database type. + // down the road it would probably make sense to split these out and find a better home for some of the + // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially + // pedantic about where precisely it should be. + + copyDatabase("ipinfo/ip_asn_sample.mmdb", tmpDir.resolve("ip_asn_sample.mmdb")); + copyDatabase("ipinfo/ip_geolocation_sample.mmdb", tmpDir.resolve("ip_geolocation_sample.mmdb")); + copyDatabase("ipinfo/asn_sample.mmdb", tmpDir.resolve("asn_sample.mmdb")); + copyDatabase("ipinfo/ip_country_sample.mmdb", tmpDir.resolve("ip_country_sample.mmdb")); + copyDatabase("ipinfo/privacy_detection_sample.mmdb", tmpDir.resolve("privacy_detection_sample.mmdb")); + + assertThat(parseDatabaseFromType("ip_asn_sample.mmdb"), is(Database.AsnV2)); + assertThat(parseDatabaseFromType("ip_geolocation_sample.mmdb"), is(Database.CityV2)); + assertThat(parseDatabaseFromType("asn_sample.mmdb"), is(Database.AsnV2)); + assertThat(parseDatabaseFromType("ip_country_sample.mmdb"), is(Database.CountryV2)); + assertThat(parseDatabaseFromType("privacy_detection_sample.mmdb"), is(Database.PrivacyDetection)); + + // additional cases where we're bailing early on types we don't support + assertThat(IpDataLookupFactories.getDatabase("ipinfo ip_country_asn_sample.mmdb"), nullValue()); + assertThat(IpDataLookupFactories.getDatabase("ipinfo privacy_detection_extended_sample.mmdb"), nullValue()); + } + + private Database parseDatabaseFromType(String databaseFile) throws IOException { + return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); + } + private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer> rowConsumer) { try (Reader reader = new Reader(pathToFile(databasePath))) { Networks networks = reader.networks(Map.class); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index c1cad60e94a9c..eec6e003f3556 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -46,9 +46,7 @@ public List routes() { new Route(GET, "/_msearch/template"), new Route(POST, "/_msearch/template"), new Route(GET, "/{index}/_msearch/template"), - new Route(POST, "/{index}/_msearch/template"), - Route.builder(GET, "/{index}/{type}/_msearch/template").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_msearch/template").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_msearch/template") ); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index 152e98f39f3b4..50a130d0fbfe8 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -48,13 +47,7 @@ public List routes() { new Route(GET, "/_search/template"), new Route(POST, "/_search/template"), new Route(GET, "/{index}/_search/template"), - new Route(POST, "/{index}/_search/template"), - Route.builder(GET, "/{index}/{type}/_search/template") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) - .build(), - Route.builder(POST, "/{index}/{type}/_search/template") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) - .build() + new Route(POST, "/{index}/_search/template") ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java index 920968cc8cbca..29c1519afa4d1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java @@ -10,14 +10,12 @@ package org.elasticsearch.reindex; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; import java.util.HashMap; @@ -40,13 +38,7 @@ public RestDeleteByQueryAction(Predicate clusterSupportsFeature) { @Override public List routes() { - return List.of( - new Route(POST, "/{index}/_delete_by_query"), - Route.builder(POST, "/{index}/{type}/_delete_by_query") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) - .build() - ); - + return List.of(new Route(POST, "/{index}/_delete_by_query")); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java index 32eb34cbf4c71..67ea34f504790 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.Script; import java.io.IOException; @@ -41,12 +40,7 @@ public RestUpdateByQueryAction(Predicate clusterSupportsFeature) { @Override public List routes() { - return List.of( - new Route(POST, "/{index}/_update_by_query"), - Route.builder(POST, "/{index}/{type}/_update_by_query") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) - .build() - ); + return List.of(new Route(POST, "/{index}/_update_by_query")); } @Override diff --git a/muted-tests.yml b/muted-tests.yml index 31f99b0ae632a..51390811e0e6c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -283,12 +283,6 @@ tests: - class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT method: testGzippedDatabase issue: https://github.com/elastic/elasticsearch/issues/113752 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=indices.split/40_routing_partition_size/more than 1} - issue: https://github.com/elastic/elasticsearch/issues/113841 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=indices.split/40_routing_partition_size/nested} - issue: https://github.com/elastic/elasticsearch/issues/113842 - class: org.elasticsearch.threadpool.SimpleThreadPoolIT method: testThreadPoolMetrics issue: https://github.com/elastic/elasticsearch/issues/108320 @@ -369,15 +363,71 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=synonyms/60_synonym_rule_get/Synonym rule not found} issue: https://github.com/elastic/elasticsearch/issues/114444 -- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests - method: testPushSpatialIntersectsEvalToSource {default} - issue: https://github.com/elastic/elasticsearch/issues/114627 -- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests - method: testPushWhereEvalToSource {default} - issue: https://github.com/elastic/elasticsearch/issues/114628 - class: org.elasticsearch.xpack.inference.integration.ModelRegistryIT method: testGetModel issue: https://github.com/elastic/elasticsearch/issues/114657 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/rest-api/usage/line_38} + issue: https://github.com/elastic/elasticsearch/issues/113694 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/10_basic/Test using the deprecated elasticsearch_version field results in a warning} + issue: https://github.com/elastic/elasticsearch/issues/114748 +- class: org.elasticsearch.xpack.eql.EqlRestIT + method: testIndexWildcardPatterns + issue: https://github.com/elastic/elasticsearch/issues/114749 +- class: org.elasticsearch.xpack.eql.EqlRestIT + method: testBadRequests + issue: https://github.com/elastic/elasticsearch/issues/114752 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/20_standard_index/enrich stats REST response structure} + issue: https://github.com/elastic/elasticsearch/issues/114753 +- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT + method: test {yaml=rrf/800_rrf_with_text_similarity_reranker_retriever/explain using rrf retriever and text-similarity} + issue: https://github.com/elastic/elasticsearch/issues/114757 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/30_tsdb_index/enrich documents over _bulk} + issue: https://github.com/elastic/elasticsearch/issues/114761 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/20_standard_index/enrich documents over _bulk via an alias} + issue: https://github.com/elastic/elasticsearch/issues/114763 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/10_basic/Test enrich crud apis} + issue: https://github.com/elastic/elasticsearch/issues/114766 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/20_standard_index/enrich documents over _bulk} + issue: https://github.com/elastic/elasticsearch/issues/114768 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/50_data_stream/enrich documents over _bulk via a data stream} + issue: https://github.com/elastic/elasticsearch/issues/114769 +- class: org.elasticsearch.xpack.eql.EqlRestValidationIT + method: testDefaultIndicesOptions + issue: https://github.com/elastic/elasticsearch/issues/114771 +- class: org.elasticsearch.xpack.enrich.EnrichIT + method: testEnrichSpecialTypes + issue: https://github.com/elastic/elasticsearch/issues/114773 +- class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT + method: testEveryActionIsEitherOperatorOnlyOrNonOperator + issue: https://github.com/elastic/elasticsearch/issues/102992 +- class: org.elasticsearch.xpack.enrich.EnrichIT + method: testDeleteExistingPipeline + issue: https://github.com/elastic/elasticsearch/issues/114775 +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/114787 +- class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests + method: testNoStream + issue: https://github.com/elastic/elasticsearch/issues/114788 +- class: org.elasticsearch.xpack.eql.EqlRestValidationIT + method: testAllowNoIndicesOption + issue: https://github.com/elastic/elasticsearch/issues/114789 +- class: org.elasticsearch.xpack.eql.EqlStatsIT + method: testEqlRestUsage + issue: https://github.com/elastic/elasticsearch/issues/114790 +- class: org.elasticsearch.xpack.eql.EqlRestIT + method: testUnicodeChars + issue: https://github.com/elastic/elasticsearch/issues/114791 +- class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderNestedDocsIT + method: testRRFExplainWithNamedRetrievers + issue: https://github.com/elastic/elasticsearch/issues/114820 # Examples: # diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index 8ce1bfdc61f6b..3a24427df24a3 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -89,7 +89,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .setting("xpack.security.enabled", "false") // geohex_grid requires gold license .setting("xpack.license.self_generated.type", "trial") - .feature(FeatureFlag.TIME_SERIES_MODE); + .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED); private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local() .name(REMOTE_CLUSTER_NAME) diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java index acdd540ca7b9d..5ada1e941266a 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java @@ -91,6 +91,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .setting("xpack.security.remote_cluster_server.ssl.enabled", "false") .setting("xpack.security.remote_cluster_client.ssl.enabled", "false") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .user("test_admin", "x-pack-test-password"); private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local() diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle index 6d94e3756eae4..3db6e2e987262 100644 --- a/qa/ccs-unavailable-clusters/build.gradle +++ b/qa/ccs-unavailable-clusters/build.gradle @@ -6,9 +6,12 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-java-rest-test' -testClusters.matching { it.name == "javaRestTest" }.configureEach { - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply plugin: 'elasticsearch.internal-java-rest-test' + + +tasks.withType(StandaloneRestIntegTestTask) { + usesDefaultDistribution() } diff --git a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index 13a77e158c343..7b42292848395 100644 --- a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.test.transport.MockTransportService; @@ -45,6 +46,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; import java.io.IOException; import java.util.Collections; @@ -61,6 +63,14 @@ public class CrossClusterSearchUnavailableClusterIT extends ESRestTestCase { private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + @Override public void tearDown() throws Exception { super.tearDown(); diff --git a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java index c68d27b883c53..e53c0564be297 100644 --- a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java +++ b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java @@ -35,6 +35,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe // The first node does not have the ingest role so we're sure ingest requests are forwarded: .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]")) .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/renovate.json b/renovate.json index 0a1d588e6332c..293a2bb262375 100644 --- a/renovate.json +++ b/renovate.json @@ -30,7 +30,7 @@ "\\s*\"?(?[^\\s:@\"]+)(?::(?[-a-zA-Z0-9.]+))?(?:@(?sha256:[a-zA-Z0-9]+))?\"?" ], "currentValueTemplate": "{{#if currentValue}}{{{currentValue}}}{{else}}latest{{/if}}", - "autoReplaceStringTemplate": "\"{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"", + "autoReplaceStringTemplate": "{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"", "datasourceTemplate": "docker" } ] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json b/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json new file mode 100644 index 0000000000000..c82b45771ac7f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json @@ -0,0 +1,38 @@ +{ + "query_rules.test": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/test-query-ruleset.html", + "description": "Tests a query ruleset to identify the rules that would match input criteria" + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query_rules/{ruleset_id}/_test", + "methods": [ + "POST" + ], + "parts": { + "ruleset_id": { + "type": "string", + "description": "The unique identifier of the ruleset to test." + } + } + } + ] + }, + "body": { + "description": "The match criteria to test against the ruleset", + "required": true + } + } +} diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java index 2b20e35019424..084e212a913b2 100644 --- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java +++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java @@ -36,6 +36,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .module("health-shards-availability") .module("data-streams") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index dfe6c9820a16a..eab51427876aa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -411,6 +411,55 @@ index param - nested array within array: - match: { hits.hits.0._source.path.to.some.3.id: [ 1000, 2000 ] } +--- +index param - nested array within array - disabled second pass: + - requires: + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + synthetic_source: + enable_second_doc_parsing_pass: false + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + path: + properties: + to: + properties: + some: + synthetic_source_keep: arrays + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }' + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.hits.0._source.name: A } + - length: { hits.hits.0._source.path.to.some: 2} + - match: { hits.hits.0._source.path.to.some.0.id: 10 } + - match: { hits.hits.0._source.path.to.some.1.id: [ 1, 3, 2] } + + --- # 112156 stored field under object with store_array_source: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml new file mode 100644 index 0000000000000..188c155e4a836 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml @@ -0,0 +1,160 @@ +setup: + - requires: + cluster_features: "mapper.vectors.bbq" + reason: 'kNN float to better-binary quantization is required' + - do: + indices.create: + index: bbq_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 64 + index: true + similarity: l2_norm + index_options: + type: bbq_hnsw + another_vector: + type: dense_vector + dims: 64 + index: true + similarity: l2_norm + index_options: + type: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "1" + body: + name: cow.jpg + vector: [300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0] + another_vector: [115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "2" + body: + name: moose.jpg + vector: [100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0] + another_vector: [50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "3" + body: + name: rabbit.jpg + vector: [111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0] + another_vector: [11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + indices.forcemerge: + index: bbq_hnsw + max_num_segments: 1 +--- +"Test knn search": + - do: + search: + index: bbq_hnsw + body: + knn: + field: vector + query_vector: [ 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + + # Depending on how things are distributed, docs 2 and 3 might be swapped + # here we verify that are last hit is always the worst one + - match: { hits.hits.2._id: "1" } + +--- +"Test bad quantization parameters": + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + element_type: byte + index: true + index_options: + type: bbq_hnsw + + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: false + index_options: + type: bbq_hnsw +--- +"Test few dimensions fail indexing": + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + dims: 42 + index: true + index_options: + type: bbq_hnsw + + - do: + indices.create: + index: dynamic_dim_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + index_options: + type: bbq_hnsw + + - do: + catch: bad_request + index: + index: dynamic_dim_bbq_hnsw + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + - do: + index: + index: dynamic_dim_bbq_hnsw + body: + vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml new file mode 100644 index 0000000000000..ed7a8dd5df65d --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml @@ -0,0 +1,165 @@ +setup: + - requires: + cluster_features: "mapper.vectors.bbq" + reason: 'kNN float to better-binary quantization is required' + - do: + indices.create: + index: bbq_flat + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 64 + index: true + similarity: l2_norm + index_options: + type: bbq_flat + another_vector: + type: dense_vector + dims: 64 + index: true + similarity: l2_norm + index_options: + type: bbq_flat + + - do: + index: + index: bbq_flat + id: "1" + body: + name: cow.jpg + vector: [300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0] + another_vector: [115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + index: + index: bbq_flat + id: "2" + body: + name: moose.jpg + vector: [100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0] + another_vector: [50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + index: + index: bbq_flat + id: "3" + body: + name: rabbit.jpg + vector: [111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0] + another_vector: [11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + indices.forcemerge: + index: bbq_flat + max_num_segments: 1 +--- +"Test knn search": + - do: + search: + index: bbq_flat + body: + knn: + field: vector + query_vector: [ 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + + # Depending on how things are distributed, docs 2 and 3 might be swapped + # here we verify that are last hit is always the worst one + - match: { hits.hits.2._id: "1" } +--- +"Test bad parameters": + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + index_options: + type: bbq_flat + m: 42 + + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + element_type: byte + index: true + index_options: + type: bbq_flat +--- +"Test few dimensions fail indexing": + # verify index creation fails + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 42 + index: true + similarity: l2_norm + index_options: + type: bbq_flat + + # verify dynamic dimension fails + - do: + indices.create: + index: dynamic_dim_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + index_options: + type: bbq_flat + + # verify index fails for odd dim vector + - do: + catch: bad_request + index: + index: dynamic_dim_bbq_flat + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + # verify that we can index an even dim vector after the odd dim vector failure + - do: + index: + index: dynamic_dim_bbq_flat + body: + vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml index ed469ffd7ff16..02576ad1b2b01 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml @@ -354,3 +354,54 @@ setup: dims: 40 index: true similarity: max_inner_product + + +--- +"Search with synthetic source": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ bit_dense_vector_synthetic_source ] + test_runner_features: capabilities + reason: "Support for bit dense vector synthetic source capability required" + - do: + indices.create: + index: test_synthetic_source + body: + mappings: + properties: + name: + type: keyword + vector1: + type: dense_vector + element_type: bit + dims: 40 + index: false + vector2: + type: dense_vector + element_type: bit + dims: 40 + index: true + similarity: l2_norm + + - do: + index: + index: test_synthetic_source + id: "1" + body: + name: cow.jpg + vector1: [2, -1, 1, 4, -3] + vector2: [2, -1, 1, 4, -3] + + - do: + indices.refresh: {} + + - do: + search: + force_synthetic_source: true + index: test_synthetic_source + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0._source.vector1: [2, -1, 1, 4, -3]} + - match: {hits.hits.0._source.vector2: [2, -1, 1, 4, -3]} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 3ca3f20917009..0647a24aa39c8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -881,7 +881,7 @@ public void testPartitionedTemplate() throws Exception { ); assertThat( eBadSettings.getMessage(), - containsString("partition size [6] should be a positive number less than the number of shards [5]") + containsString("partition size [6] should be a positive number less than the number of routing shards [5]") ); // provide an invalid mapping for a partitioned index @@ -913,7 +913,7 @@ public void testPartitionedTemplate() throws Exception { assertThat( eBadIndex.getMessage(), - containsString("partition size [6] should be a positive number less than the number of shards [5]") + containsString("partition size [6] should be a positive number less than the number of routing shards [5]") ); // finally, create a valid index diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index 1787b4f784574..a8e2ca818d3f4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -88,11 +88,11 @@ private static String format(ZonedDateTime date, String pattern) { private IndexRequestBuilder indexDoc(String idx, ZonedDateTime date, int value) throws Exception { return prepareIndex(idx).setSource( jsonBuilder().startObject() - .timeField("date", date) + .timestampField("date", date) .field("value", value) .startArray("dates") - .timeValue(date) - .timeValue(date.plusMonths(1).plusDays(1)) + .timestampValue(date) + .timestampValue(date.plusMonths(1).plusDays(1)) .endArray() .endObject() ); @@ -103,10 +103,10 @@ private IndexRequestBuilder indexDoc(int month, int day, int value) throws Excep jsonBuilder().startObject() .field("value", value) .field("constant", 1) - .timeField("date", date(month, day)) + .timestampField("date", date(month, day)) .startArray("dates") - .timeValue(date(month, day)) - .timeValue(date(month + 1, day + 1)) + .timestampValue(date(month, day)) + .timestampValue(date(month + 1, day + 1)) .endArray() .endObject() ); @@ -162,53 +162,53 @@ private void getMultiSortDocs(List builders) throws IOExcep for (int i = 1; i <= 3; i++) { builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 1)).field("l", 1).field("d", i).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 1)).field("l", 1).field("d", i).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 2)).field("l", 2).field("d", i).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 2)).field("l", 2).field("d", i).endObject() ) ); } builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 3)).field("l", 3).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 3)).field("l", 3).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 3).plusHours(1)).field("l", 3).field("d", 2).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 3).plusHours(1)).field("l", 3).field("d", 2).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 4)).field("l", 3).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 4)).field("l", 3).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 4).plusHours(2)).field("l", 3).field("d", 3).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 4).plusHours(2)).field("l", 3).field("d", 3).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 5)).field("l", 5).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 5)).field("l", 5).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 5).plusHours(12)).field("l", 5).field("d", 2).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 5).plusHours(12)).field("l", 5).field("d", 2).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 6)).field("l", 5).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 6)).field("l", 5).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 7)).field("l", 5).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 7)).field("l", 5).field("d", 1).endObject() ) ); } @@ -997,7 +997,7 @@ public void testSingleValueWithTimeZone() throws Exception { IndexRequestBuilder[] reqs = new IndexRequestBuilder[5]; ZonedDateTime date = date("2014-03-11T00:00:00+00:00"); for (int i = 0; i < reqs.length; i++) { - reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().timeField("date", date).endObject()); + reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().timestampField("date", date).endObject()); date = date.plusHours(1); } indexRandom(true, reqs); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 0afc479474814..778be4ee0705f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -63,7 +63,7 @@ private void prepareIndex(ZonedDateTime date, int numHours, int stepSizeHours, i IndexRequestBuilder[] reqs = new IndexRequestBuilder[numHours]; for (int i = idxIdStart; i < idxIdStart + reqs.length; i++) { reqs[i - idxIdStart] = prepareIndex("idx2").setId("" + i) - .setSource(jsonBuilder().startObject().timeField("date", date).endObject()); + .setSource(jsonBuilder().startObject().timestampField("date", date).endObject()); date = date.plusHours(stepSizeHours); } indexRandom(true, reqs); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java index 6e9a9305eaf4e..afa3ad9d7e737 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java @@ -58,10 +58,10 @@ private static IndexRequestBuilder indexDoc(int month, int day, int value) throw return prepareIndex("idx").setSource( jsonBuilder().startObject() .field("value", value) - .timeField("date", date(month, day)) + .timestampField("date", date(month, day)) .startArray("dates") - .timeValue(date(month, day)) - .timeValue(date(month + 1, day + 1)) + .timestampValue(date(month, day)) + .timestampValue(date(month + 1, day + 1)) .endArray() .endObject() ); @@ -620,8 +620,8 @@ public void testScriptCaching() throws Exception { ); indexRandom( true, - prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().timeField("date", date(1, 1)).endObject()), - prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().timeField("date", date(2, 1)).endObject()) + prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().timestampField("date", date(1, 1)).endObject()), + prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().timestampField("date", date(2, 1)).endObject()) ); // Make sure we are starting with a clear cache diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 70b748c86ec96..414a6c6ba66a6 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -7,7 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; import org.elasticsearch.plugins.internal.RestExtension; /** The Elasticsearch Server Module. */ @@ -445,14 +444,16 @@ org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat, org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat, org.elasticsearch.index.codec.postings.ES812PostingsFormat; - provides org.apache.lucene.codecs.DocValuesFormat with ES87TSDBDocValuesFormat; + provides org.apache.lucene.codecs.DocValuesFormat with org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; provides org.apache.lucene.codecs.KnnVectorsFormat with org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat, org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat, org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat, - org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat; + org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat, + org.elasticsearch.index.codec.vectors.ES816BinaryQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.ES816HnswBinaryQuantizedVectorsFormat; provides org.apache.lucene.codecs.Codec with diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 03186e63240e5..3cb4695e867df 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -242,6 +242,8 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_CACHED_STRING_SERIALIZATION = def(8_766_00_0); public static final TransportVersion CHUNK_SENTENCE_OVERLAP_SETTING_ADDED = def(8_767_00_0); public static final TransportVersion OPT_IN_ESQL_CCS_EXECUTION_INFO = def(8_768_00_0); + public static final TransportVersion QUERY_RULE_TEST_API = def(8_769_00_0); + public static final TransportVersion ESQL_PER_AGGREGATE_FILTER = def(8_770_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 2d72f5d71ccda..08558b48c08b3 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -352,9 +352,7 @@ import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import org.elasticsearch.rest.action.admin.indices.RestSimulateIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestSimulateTemplateAction; -import org.elasticsearch.rest.action.admin.indices.RestSyncedFlushAction; import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction; -import org.elasticsearch.rest.action.admin.indices.RestUpgradeActionDeprecated; import org.elasticsearch.rest.action.admin.indices.RestValidateQueryAction; import org.elasticsearch.rest.action.cat.AbstractCatAction; import org.elasticsearch.rest.action.cat.RestAliasAction; @@ -916,7 +914,6 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< registerHandler.accept(new RestRefreshAction()); registerHandler.accept(new RestFlushAction()); - registerHandler.accept(new RestSyncedFlushAction()); registerHandler.accept(new RestForceMergeAction()); registerHandler.accept(new RestClearIndicesCacheAction()); registerHandler.accept(new RestResolveClusterAction()); @@ -1003,8 +1000,6 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< registerHandler.accept(new RestAnalyzeIndexDiskUsageAction()); registerHandler.accept(new RestFieldUsageStatsAction()); - registerHandler.accept(new RestUpgradeActionDeprecated()); - // Desired nodes registerHandler.accept(new RestGetDesiredNodesAction()); registerHandler.accept(new RestUpdateDesiredNodesAction(clusterSupportsFeature)); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index 7b344a4c25a1b..45ee00a98c2e2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -92,17 +92,12 @@ public Iterator toXContentChunked(ToXContent.Params outerP if (emitState(outerParams)) { deprecationLogger.critical(DeprecationCategory.API, "reroute_cluster_state", STATE_FIELD_DEPRECATION_MESSAGE); } - return toXContentChunkedV7(outerParams); - } - - @Override - public Iterator toXContentChunkedV7(ToXContent.Params params) { - return ChunkedToXContent.builder(params).object(b -> { + return ChunkedToXContent.builder(outerParams).object(b -> { b.field(ACKNOWLEDGED_KEY, isAcknowledged()); - if (emitState(params)) { + if (emitState(outerParams)) { b.xContentObject("state", state); } - if (params.paramAsBoolean("explain", false)) { + if (outerParams.paramAsBoolean("explain", false)) { b.append(explanations); } }); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index 9ffef1f178f44..b855f2cee7613 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -84,7 +85,7 @@ protected void masterOperation( String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request); Map> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices()); Map indicesAndFilters = new HashMap<>(); - Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); + Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); for (String index : concreteIndices) { final AliasFilter aliasFilter = indicesService.buildAliasFilter(clusterState, index, indicesAndAliases); final String[] aliases = indexNameExpressionResolver.indexAliases( diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java index 6fe8432c31ccc..d942c4347960a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java @@ -79,7 +79,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("index_name", info.indexName); builder.field("index_uuid", info.indexUUID); - builder.timeField("creation_date_millis", "creation_date", info.creationDateMillis); + builder.timestampFieldsFromUnixEpochMillis("creation_date_millis", "creation_date", info.creationDateMillis); builder.array("node_ids", info.nodeIds.toArray(new String[0])); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 5c5c71bc002b3..f5c100b7884bb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -565,8 +566,8 @@ static void resolveIndices( if (names.length == 1 && (Metadata.ALL.equals(names[0]) || Regex.isMatchAllPattern(names[0]))) { names = new String[] { "**" }; } - Set resolvedIndexAbstractions = resolver.resolveExpressions(clusterState, indicesOptions, true, names); - for (String s : resolvedIndexAbstractions) { + Set resolvedIndexAbstractions = resolver.resolveExpressions(clusterState, indicesOptions, true, names); + for (ResolvedExpression s : resolvedIndexAbstractions) { enrichIndexAbstraction(clusterState, s, indices, aliases, dataStreams); } indices.sort(Comparator.comparing(ResolvedIndexAbstraction::getName)); @@ -597,12 +598,12 @@ private static void mergeResults( private static void enrichIndexAbstraction( ClusterState clusterState, - String indexAbstraction, + ResolvedExpression indexAbstraction, List indices, List aliases, List dataStreams ) { - IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction); + IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction.resource()); if (ia != null) { switch (ia.getType()) { case CONCRETE_INDEX -> { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java index 47abda4fabcde..347376a918d4c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java @@ -69,7 +69,7 @@ public FieldUsageStats getStats() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(Fields.TRACKING_ID, trackingId); - builder.timeField(Fields.TRACKING_STARTED_AT_MILLIS, Fields.TRACKING_STARTED_AT, trackingStartTime); + builder.timestampFieldsFromUnixEpochMillis(Fields.TRACKING_STARTED_AT_MILLIS, Fields.TRACKING_STARTED_AT, trackingStartTime); builder.startObject(Fields.ROUTING) .field(Fields.STATE, shardRouting.state()) .field(Fields.PRIMARY, shardRouting.primary()) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index ec8eb4babfdac..5e3799cd14518 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -274,7 +274,7 @@ public static Template resolveTemplate( Settings result = provider.getAdditionalIndexSettings( indexName, template.getDataStreamTemplate() != null ? indexName : null, - template.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(template), + metadata.retrieveIndexModeFromTemplate(template), simulatedState.getMetadata(), now, templateSettings, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 4e9830fe0d14e..e01f364712676 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; @@ -133,7 +134,7 @@ protected void doExecute(Task task, ValidateQueryRequest request, ActionListener @Override protected ShardValidateQueryRequest newShardRequest(int numShards, ShardRouting shard, ValidateQueryRequest request) { final ClusterState clusterState = clusterService.state(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, shard.getIndexName(), indicesAndAliases); return new ShardValidateQueryRequest(shard.shardId(), aliasFilter, request); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java index f433e937dbe5d..a5a38a288d342 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java @@ -18,12 +18,14 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.function.Supplier; +import static org.elasticsearch.common.xcontent.XContentElasticsearchExtension.DEFAULT_FORMATTER; import static org.elasticsearch.ingest.CompoundProcessor.PIPELINE_ORIGIN_EXCEPTION_HEADER; import static org.elasticsearch.ingest.CompoundProcessor.PROCESSOR_TAG_EXCEPTION_HEADER; import static org.elasticsearch.ingest.CompoundProcessor.PROCESSOR_TYPE_EXCEPTION_HEADER; @@ -84,7 +86,7 @@ private static XContentBuilder createSource(IndexRequest source, Exception excep XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); { - builder.timeField("@timestamp", timeSupplier.get()); + builder.field("@timestamp", DEFAULT_FORMATTER.format(Instant.ofEpochMilli(timeSupplier.get()))); builder.startObject("document"); { if (source.id() != null) { diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java index 2352628264394..94c294435acd3 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java @@ -123,7 +123,7 @@ public XContentBuilder toXContent( builder.field(MANAGED_BY_LIFECYCLE_FIELD.getPreferredName(), managedByLifecycle); if (managedByLifecycle) { if (indexCreationDate != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( INDEX_CREATION_DATE_MILLIS_FIELD.getPreferredName(), INDEX_CREATION_DATE_FIELD.getPreferredName(), indexCreationDate @@ -134,7 +134,11 @@ public XContentBuilder toXContent( ); } if (rolloverDate != null) { - builder.timeField(ROLLOVER_DATE_MILLIS_FIELD.getPreferredName(), ROLLOVER_DATE_FIELD.getPreferredName(), rolloverDate); + builder.timestampFieldsFromUnixEpochMillis( + ROLLOVER_DATE_MILLIS_FIELD.getPreferredName(), + ROLLOVER_DATE_FIELD.getPreferredName(), + rolloverDate + ); builder.field(TIME_SINCE_ROLLOVER_FIELD.getPreferredName(), getTimeSinceRollover(nowSupplier).toHumanReadableString(2)); } if (generationDateMillis != null) { diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index 9c82d032014f2..84c6df7b8a66f 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -109,7 +110,7 @@ protected boolean resolveIndex(ExplainRequest request) { @Override protected void resolveRequest(ClusterState state, InternalRequest request) { - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(state, request.request().index()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(state, request.request().index()); final AliasFilter aliasFilter = searchService.buildAliasFilter(state, request.concreteIndex(), indicesAndAliases); request.request().filteringAlias(aliasFilter); } diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 99eac250641ae..fb4b3907d2bfd 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.admin.indices.refresh.TransportShardRefreshAction; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.replication.BasicReplicationRequest; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.client.internal.node.NodeClient; @@ -284,11 +285,11 @@ private void tryGetFromTranslog(GetRequest request, IndexShard indexShard, Disco } else { assert r.segmentGeneration() > -1L; assert r.primaryTerm() > Engine.UNKNOWN_PRIMARY_TERM; - indexShard.waitForPrimaryTermAndGeneration( - r.primaryTerm(), - r.segmentGeneration(), - listener.delegateFailureAndWrap((ll, aLong) -> super.asyncShardOperation(request, shardId, ll)) + final ActionListener termAndGenerationListener = ContextPreservingActionListener.wrapPreservingContext( + listener.delegateFailureAndWrap((ll, aLong) -> super.asyncShardOperation(request, shardId, ll)), + threadPool.getThreadContext() ); + indexShard.waitForPrimaryTermAndGeneration(r.primaryTerm(), r.segmentGeneration(), termAndGenerationListener); } } }), TransportGetFromTranslogAction.Response::new, getExecutor(request, shardId)) diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 1645a378446a4..b5864f64a7824 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.GroupShardsIterator; @@ -110,6 +111,7 @@ import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.stream.Collectors; import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; @@ -203,7 +205,7 @@ public TransportSearchAction( private Map buildPerIndexOriginalIndices( ClusterState clusterState, - Set indicesAndAliases, + Set indicesAndAliases, String[] indices, IndicesOptions indicesOptions ) { @@ -211,6 +213,9 @@ private Map buildPerIndexOriginalIndices( var blocks = clusterState.blocks(); // optimization: mostly we do not have any blocks so there's no point in the expensive per-index checking boolean hasBlocks = blocks.global().isEmpty() == false || blocks.indices().isEmpty() == false; + // Get a distinct set of index abstraction names present from the resolved expressions to help with the reverse resolution from + // concrete index to the expression that produced it. + Set indicesAndAliasesResources = indicesAndAliases.stream().map(ResolvedExpression::resource).collect(Collectors.toSet()); for (String index : indices) { if (hasBlocks) { blocks.indexBlockedRaiseException(ClusterBlockLevel.READ, index); @@ -227,8 +232,8 @@ private Map buildPerIndexOriginalIndices( String[] finalIndices = Strings.EMPTY_ARRAY; if (aliases == null || aliases.length == 0 - || indicesAndAliases.contains(index) - || hasDataStreamRef(clusterState, indicesAndAliases, index)) { + || indicesAndAliasesResources.contains(index) + || hasDataStreamRef(clusterState, indicesAndAliasesResources, index)) { finalIndices = new String[] { index }; } if (aliases != null) { @@ -247,7 +252,11 @@ private static boolean hasDataStreamRef(ClusterState clusterState, Set i return indicesAndAliases.contains(ret.getParentDataStream().getName()); } - Map buildIndexAliasFilters(ClusterState clusterState, Set indicesAndAliases, Index[] concreteIndices) { + Map buildIndexAliasFilters( + ClusterState clusterState, + Set indicesAndAliases, + Index[] concreteIndices + ) { final Map aliasFilterMap = new HashMap<>(); for (Index index : concreteIndices) { clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index.getName()); @@ -1237,7 +1246,10 @@ private void executeSearch( } else { final Index[] indices = resolvedIndices.getConcreteLocalIndices(); concreteLocalIndices = Arrays.stream(indices).map(Index::getName).toArray(String[]::new); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, searchRequest.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + clusterState, + searchRequest.indices() + ); aliasFilter = buildIndexAliasFilters(clusterState, indicesAndAliases, indices); aliasFilter.putAll(remoteAliasMap); localShardIterators = getLocalShardsIterator( @@ -1810,7 +1822,7 @@ List getLocalShardsIterator( ClusterState clusterState, SearchRequest searchRequest, String clusterAlias, - Set indicesAndAliases, + Set indicesAndAliases, String[] concreteIndices ) { var routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(), searchRequest.indices()); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java index f418b5617b2a1..b94bd95c93d8a 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.index.Index; @@ -127,7 +128,10 @@ public void searchShards(Task task, SearchShardsRequest searchShardsRequest, Act searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices, null), listener.delegateFailureAndWrap((delegate, searchRequest) -> { Index[] concreteIndices = resolvedIndices.getConcreteLocalIndices(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, searchRequest.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + clusterState, + searchRequest.indices() + ); final Map aliasFilters = transportSearchAction.buildIndexAliasFilters( clusterState, indicesAndAliases, diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java index cb98cd4b2f535..ac96a2d55bc71 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java @@ -228,7 +228,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.endObject(); - builder.timeField("oldest_start_time_millis", "oldest_start_time", firstStartTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("oldest_start_time_millis", "oldest_start_time", firstStartTimeMillis); return builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index c371ff4d37a05..fe144135d42bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -180,7 +180,7 @@ public Iterator toXContentChunked(ToXContent.Params ignore builder.value(snapshot.getName()); } builder.endArray(); - builder.timeField("start_time_millis", "start_time", entry.startTime); + builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", entry.startTime); builder.field("repository_state_id", entry.repositoryStateId); builder.field("state", entry.state); } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index c32175fc9367d..d82a31720d6d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -1404,7 +1404,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endArray(); - builder.timeField("start_time_millis", "start_time", startTime); + builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", startTime); builder.field("repository_state_id", repositoryStateId); builder.startArray("shards"); { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java index 783145d3618f1..320be8acb0af9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java @@ -434,7 +434,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.startObject(); builder.field(INDEX_KEY); index.toXContent(builder, params); - builder.timeField(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis); + builder.timestampFieldsFromUnixEpochMillis(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis); return builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 9760d84c67c5b..23e8e49aa16db 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2265,7 +2265,7 @@ IndexMetadata build(boolean repair) { "routing partition size [" + routingPartitionSize + "] should be a positive number" - + " less than the number of shards [" + + " less than the number of routing shards [" + getRoutingNumShards() + "] for [" + index diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 2229166a2d779..eaf54034b22e0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -74,6 +74,15 @@ public IndexNameExpressionResolver(ThreadContext threadContext, SystemIndices sy this.systemIndices = Objects.requireNonNull(systemIndices, "System Indices must not be null"); } + /** + * This contains the resolved expression in the form of the resource. + * Soon it will facilitate the index component selector. + * @param resource the resolved resolvedExpression + */ + public record ResolvedExpression(String resource) { + + } + /** * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options * are encapsulated in the specified request. @@ -191,8 +200,9 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - final Collection expressions = resolveExpressions(context, indexExpressions); + final Collection expressions = resolveExpressions(context, indexExpressions); return expressions.stream() + .map(ResolvedExpression::resource) .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) .filter(ia -> ia.getType() == Type.DATA_STREAM) @@ -221,10 +231,11 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit getNetNewSystemIndexPredicate() ); - final Collection expressions = resolveExpressions(context, request.index()); + final Collection expressions = resolveExpressions(context, request.index()); if (expressions.size() == 1) { - IndexAbstraction ia = state.metadata().getIndicesLookup().get(expressions.iterator().next()); + ResolvedExpression resolvedExpression = expressions.iterator().next(); + IndexAbstraction ia = state.metadata().getIndicesLookup().get(resolvedExpression.resource()); if (ia.getType() == Type.ALIAS) { Index writeIndex = ia.getWriteIndex(); if (writeIndex == null) { @@ -246,14 +257,14 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit } } - protected static Collection resolveExpressions(Context context, String... expressions) { + protected static Collection resolveExpressions(Context context, String... expressions) { if (context.getOptions().expandWildcardExpressions() == false) { if (expressions == null || expressions.length == 0 || expressions.length == 1 && Metadata.ALL.equals(expressions[0])) { return List.of(); } else { return ExplicitResourceNameFilter.filterUnavailable( context, - DateMathExpressionResolver.resolve(context, List.of(expressions)) + DateMathExpressionResolver.resolve(context, Arrays.stream(expressions).map(ResolvedExpression::new).toList()) ); } } else { @@ -264,7 +275,10 @@ protected static Collection resolveExpressions(Context context, String.. } else { return WildcardExpressionResolver.resolve( context, - ExplicitResourceNameFilter.filterUnavailable(context, DateMathExpressionResolver.resolve(context, List.of(expressions))) + ExplicitResourceNameFilter.filterUnavailable( + context, + DateMathExpressionResolver.resolve(context, Arrays.stream(expressions).map(ResolvedExpression::new).toList()) + ) ); } } @@ -339,12 +353,12 @@ String[] concreteIndexNames(Context context, String... indexExpressions) { } Index[] concreteIndices(Context context, String... indexExpressions) { - final Collection expressions = resolveExpressions(context, indexExpressions); + final Collection expressions = resolveExpressions(context, indexExpressions); final Set concreteIndicesResult = Sets.newLinkedHashSetWithExpectedSize(expressions.size()); final Map indicesLookup = context.getState().metadata().getIndicesLookup(); - for (String expression : expressions) { - final IndexAbstraction indexAbstraction = indicesLookup.get(expression); + for (ResolvedExpression resolvedExpression : expressions) { + final IndexAbstraction indexAbstraction = indicesLookup.get(resolvedExpression.resource()); assert indexAbstraction != null; if (indexAbstraction.getType() == Type.ALIAS && context.isResolveToWriteIndex()) { Index writeIndex = indexAbstraction.getWriteIndex(); @@ -378,7 +392,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) { throw new IllegalArgumentException( indexAbstraction.getType().getDisplayName() + " [" - + expression + + resolvedExpression.resource() + "] has more than one index associated with it " + Arrays.toString(indexNames) + ", can't execute a single index op" @@ -642,7 +656,7 @@ public Index concreteSingleIndex(ClusterState state, IndicesRequest request) { * Utility method that allows to resolve an index expression to its corresponding single write index. * * @param state the cluster state containing all the data to resolve to expression to a concrete index - * @param request The request that defines how the an alias or an index need to be resolved to a concrete index + * @param request The request that defines how an alias or an index need to be resolved to a concrete index * and the expression that can be resolved to an alias or an index name. * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index * @return the write index obtained as a result of the index resolution @@ -734,7 +748,7 @@ public static String resolveDateMathExpression(String dateExpression, long time) /** * Resolve an array of expressions to the set of indices and aliases that these expressions match. */ - public Set resolveExpressions(ClusterState state, String... expressions) { + public Set resolveExpressions(ClusterState state, String... expressions) { return resolveExpressions(state, IndicesOptions.lenientExpandOpen(), false, expressions); } @@ -743,7 +757,7 @@ public Set resolveExpressions(ClusterState state, String... expressions) * If {@param preserveDataStreams} is {@code true}, datastreams that are covered by the wildcards from the * {@param expressions} are returned as-is, without expanding them further to their respective backing indices. */ - public Set resolveExpressions( + public Set resolveExpressions( ClusterState state, IndicesOptions indicesOptions, boolean preserveDataStreams, @@ -760,10 +774,10 @@ public Set resolveExpressions( getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - Collection resolved = resolveExpressions(context, expressions); - if (resolved instanceof Set) { + Collection resolved = resolveExpressions(context, expressions); + if (resolved instanceof Set) { // unmodifiable without creating a new collection as it might contain many items - return Collections.unmodifiableSet((Set) resolved); + return Collections.unmodifiableSet((Set) resolved); } else { return Set.copyOf(resolved); } @@ -776,7 +790,7 @@ public Set resolveExpressions( * the index itself - null is returned. Returns {@code null} if no filtering is required. * NOTE: The provided expressions must have been resolved already via {@link #resolveExpressions}. */ - public String[] filteringAliases(ClusterState state, String index, Set resolvedExpressions) { + public String[] filteringAliases(ClusterState state, String index, Set resolvedExpressions) { return indexAliases(state, index, AliasMetadata::filteringRequired, DataStreamAlias::filteringRequired, false, resolvedExpressions); } @@ -802,39 +816,39 @@ public String[] indexAliases( Predicate requiredAlias, Predicate requiredDataStreamAlias, boolean skipIdentity, - Set resolvedExpressions + Set resolvedExpressions ) { - if (isAllIndices(resolvedExpressions)) { + if (isAllIndicesExpression(resolvedExpressions)) { return null; } - + Set resources = resolvedExpressions.stream().map(ResolvedExpression::resource).collect(Collectors.toSet()); final IndexMetadata indexMetadata = state.metadata().getIndices().get(index); if (indexMetadata == null) { // Shouldn't happen throw new IndexNotFoundException(index); } - if (skipIdentity == false && resolvedExpressions.contains(index)) { + if (skipIdentity == false && resources.contains(index)) { return null; } IndexAbstraction ia = state.metadata().getIndicesLookup().get(index); DataStream dataStream = ia.getParentDataStream(); if (dataStream != null) { - if (skipIdentity == false && resolvedExpressions.contains(dataStream.getName())) { + if (skipIdentity == false && resources.contains(dataStream.getName())) { // skip the filters when the request targets the data stream name return null; } Map dataStreamAliases = state.metadata().dataStreamAliases(); List aliasesForDataStream; - if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) { + if (iterateIndexAliases(dataStreamAliases.size(), resources.size())) { aliasesForDataStream = dataStreamAliases.values() .stream() - .filter(dataStreamAlias -> resolvedExpressions.contains(dataStreamAlias.getName())) + .filter(dataStreamAlias -> resources.contains(dataStreamAlias.getName())) .filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName())) .toList(); } else { - aliasesForDataStream = resolvedExpressions.stream() + aliasesForDataStream = resources.stream() .map(dataStreamAliases::get) .filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName())) .toList(); @@ -859,18 +873,15 @@ public String[] indexAliases( } else { final Map indexAliases = indexMetadata.getAliases(); final AliasMetadata[] aliasCandidates; - if (iterateIndexAliases(indexAliases.size(), resolvedExpressions.size())) { + if (iterateIndexAliases(indexAliases.size(), resources.size())) { // faster to iterate indexAliases aliasCandidates = indexAliases.values() .stream() - .filter(aliasMetadata -> resolvedExpressions.contains(aliasMetadata.alias())) + .filter(aliasMetadata -> resources.contains(aliasMetadata.alias())) .toArray(AliasMetadata[]::new); } else { // faster to iterate resolvedExpressions - aliasCandidates = resolvedExpressions.stream() - .map(indexAliases::get) - .filter(Objects::nonNull) - .toArray(AliasMetadata[]::new); + aliasCandidates = resources.stream().map(indexAliases::get).filter(Objects::nonNull).toArray(AliasMetadata[]::new); } List aliases = null; for (AliasMetadata aliasMetadata : aliasCandidates) { @@ -909,12 +920,7 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - final Collection resolvedExpressions = resolveExpressions(context, expressions); - - // TODO: it appears that this can never be true? - if (isAllIndices(resolvedExpressions)) { - return resolveSearchRoutingAllIndices(state.metadata(), routing); - } + final Collection resolvedExpressions = resolveExpressions(context, expressions); Map> routings = null; Set paramRouting = null; @@ -924,8 +930,8 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab paramRouting = Sets.newHashSet(Strings.splitStringByCommaToArray(routing)); } - for (String expression : resolvedExpressions) { - IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(expression); + for (ResolvedExpression resolvedExpression : resolvedExpressions) { + IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(resolvedExpression.resource); if (indexAbstraction != null && indexAbstraction.getType() == Type.ALIAS) { for (Index index : indexAbstraction.getIndices()) { String concreteIndex = index.getName(); @@ -963,7 +969,7 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab } } else { // Index - routings = collectRoutings(routings, paramRouting, norouting, expression); + routings = collectRoutings(routings, paramRouting, norouting, resolvedExpression.resource()); } } @@ -1009,6 +1015,17 @@ public static Map> resolveSearchRoutingAllIndices(Metadata m return null; } + /** + * Identifies whether the array containing index names given as argument refers to all indices + * The empty or null array identifies all indices + * + * @param aliasesOrIndices the array containing index names + * @return true if the provided array maps to all indices, false otherwise + */ + public static boolean isAllIndicesExpression(Collection aliasesOrIndices) { + return isAllIndices(aliasesOrIndices.stream().map(ResolvedExpression::resource).toList()); + } + /** * Identifies whether the array containing index names given as argument refers to all indices * The empty or null array identifies all indices @@ -1249,8 +1266,8 @@ private WildcardExpressionResolver() { * Returns all the indices, datastreams, and aliases, considering the open/closed, system, and hidden context parameters. * Depending on the context, returns the names of the datastreams themselves or their backing indices. */ - public static Collection resolveAll(Context context) { - List concreteIndices = resolveEmptyOrTrivialWildcard(context); + public static Collection resolveAll(Context context) { + List concreteIndices = resolveEmptyOrTrivialWildcard(context); if (context.includeDataStreams() == false && context.getOptions().ignoreAliases()) { return concreteIndices; @@ -1265,7 +1282,7 @@ public static Collection resolveAll(Context context) { .filter(ia -> shouldIncludeIfDataStream(ia, context) || shouldIncludeIfAlias(ia, context)) .filter(ia -> ia.isSystem() == false || context.systemIndexAccessPredicate.test(ia.getName())); - Set resolved = expandToOpenClosed(context, ias).collect(Collectors.toSet()); + Set resolved = expandToOpenClosed(context, ias).collect(Collectors.toSet()); resolved.addAll(concreteIndices); return resolved; } @@ -1293,17 +1310,17 @@ private static boolean shouldIncludeIfAlias(IndexAbstraction ia, IndexNameExpres * ultimately returned, instead of the alias or datastream name * */ - public static Collection resolve(Context context, List expressions) { + public static Collection resolve(Context context, List expressions) { ExpressionList expressionList = new ExpressionList(context, expressions); // fast exit if there are no wildcards to evaluate if (expressionList.hasWildcard() == false) { return expressions; } - Set result = new HashSet<>(); + Set result = new HashSet<>(); for (ExpressionList.Expression expression : expressionList) { if (expression.isWildcard()) { Stream matchingResources = matchResourcesToWildcard(context, expression.get()); - Stream matchingOpenClosedNames = expandToOpenClosed(context, matchingResources); + Stream matchingOpenClosedNames = expandToOpenClosed(context, matchingResources); AtomicBoolean emptyWildcardExpansion = new AtomicBoolean(false); if (context.getOptions().allowNoIndices() == false) { emptyWildcardExpansion.set(true); @@ -1319,9 +1336,9 @@ public static Collection resolve(Context context, List expressio } } else { if (expression.isExclusion()) { - result.remove(expression.get()); + result.remove(new ResolvedExpression(expression.get())); } else { - result.add(expression.get()); + result.add(expression.resolvedExpression()); } } } @@ -1412,13 +1429,13 @@ private static Map filterIndicesLookupForSuffixWildcar * Data streams and aliases are interpreted to refer to multiple indices, * then all index resources are filtered by their open/closed status. */ - private static Stream expandToOpenClosed(Context context, Stream resources) { + private static Stream expandToOpenClosed(Context context, Stream resources) { final IndexMetadata.State excludeState = excludeState(context.getOptions()); return resources.flatMap(indexAbstraction -> { if (context.isPreserveAliases() && indexAbstraction.getType() == Type.ALIAS) { - return Stream.of(indexAbstraction.getName()); + return Stream.of(new ResolvedExpression(indexAbstraction.getName())); } else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) { - return Stream.of(indexAbstraction.getName()); + return Stream.of(new ResolvedExpression(indexAbstraction.getName())); } else { Stream indicesStateStream = Stream.of(); if (shouldIncludeRegularIndices(context.getOptions())) { @@ -1434,18 +1451,20 @@ private static Stream expandToOpenClosed(Context context, Stream indexMeta.getState() != excludeState); } - return indicesStateStream.map(indexMeta -> indexMeta.getIndex().getName()); + return indicesStateStream.map(indexMeta -> new ResolvedExpression(indexMeta.getIndex().getName())); } }); } - private static List resolveEmptyOrTrivialWildcard(Context context) { + private static List resolveEmptyOrTrivialWildcard(Context context) { final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices(context.getOptions(), context.getState().metadata()); + Stream result; if (context.systemIndexAccessLevel == SystemIndexAccessLevel.ALL) { - return List.of(allIndices); + result = Arrays.stream(allIndices); } else { - return resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(context, allIndices); + result = resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(context, allIndices).stream(); } + return result.map(ResolvedExpression::new).toList(); } private static List resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(Context context, String[] allIndices) { @@ -1507,8 +1526,8 @@ private DateMathExpressionResolver() { // utility class } - public static List resolve(Context context, List expressions) { - List result = new ArrayList<>(expressions.size()); + public static List resolve(Context context, List expressions) { + List result = new ArrayList<>(expressions.size()); for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) { result.add(resolveExpression(expression, context::getStartTime)); } @@ -1519,13 +1538,15 @@ static String resolveExpression(String expression) { return resolveExpression(expression, System::currentTimeMillis); } - static String resolveExpression(ExpressionList.Expression expression, LongSupplier getTime) { + static ResolvedExpression resolveExpression(ExpressionList.Expression expression, LongSupplier getTime) { + String result; if (expression.isExclusion()) { // accepts date-math exclusions that are of the form "-<...{}>", i.e. the "-" is outside the "<>" date-math template - return "-" + resolveExpression(expression.get(), getTime); + result = "-" + resolveExpression(expression.get(), getTime); } else { - return resolveExpression(expression.get(), getTime); + result = resolveExpression(expression.get(), getTime); } + return new ResolvedExpression(result); } static String resolveExpression(String expression, LongSupplier getTime) { @@ -1687,25 +1708,26 @@ private ExplicitResourceNameFilter() { * Returns an expression list with "unavailable" (missing or not acceptable) resource names filtered out. * Only explicit resource names are considered for filtering. Wildcard and exclusion expressions are kept in. */ - public static List filterUnavailable(Context context, List expressions) { + public static List filterUnavailable(Context context, List expressions) { ensureRemoteIndicesRequireIgnoreUnavailable(context.getOptions(), expressions); - List result = new ArrayList<>(expressions.size()); + List result = new ArrayList<>(expressions.size()); for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) { validateAliasOrIndex(expression); - if (expression.isWildcard() || expression.isExclusion() || ensureAliasOrIndexExists(context, expression.get())) { - result.add(expression.expression()); + if (expression.isWildcard() || expression.isExclusion() || ensureAliasOrIndexExists(context, expression)) { + result.add(expression.resolvedExpression()); } } return result; } /** - * This returns `true` if the given {@param name} is of a resource that exists. - * Otherwise, it returns `false` if the `ignore_unvailable` option is `true`, or, if `false`, it throws a "not found" type of + * This returns `true` if the given {@param resolvedExpression} is of a resource that exists. + * Otherwise, it returns `false` if the `ignore_unavailable` option is `true`, or, if `false`, it throws a "not found" type of * exception. */ @Nullable - private static boolean ensureAliasOrIndexExists(Context context, String name) { + private static boolean ensureAliasOrIndexExists(Context context, ExpressionList.Expression expression) { + String name = expression.get(); boolean ignoreUnavailable = context.getOptions().ignoreUnavailable(); IndexAbstraction indexAbstraction = context.getState().getMetadata().getIndicesLookup().get(name); if (indexAbstraction == null) { @@ -1737,32 +1759,37 @@ private static boolean ensureAliasOrIndexExists(Context context, String name) { } private static void validateAliasOrIndex(ExpressionList.Expression expression) { - if (Strings.isEmpty(expression.expression())) { - throw notFoundException(expression.expression()); + if (Strings.isEmpty(expression.resolvedExpression().resource())) { + throw notFoundException(expression.get()); } // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API // does not exist and the path is interpreted as an expression. If the expression begins with an underscore, // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown // if the expression can't be found. - if (expression.expression().charAt(0) == '_') { - throw new InvalidIndexNameException(expression.expression(), "must not start with '_'."); + if (expression.resolvedExpression().resource().charAt(0) == '_') { + throw new InvalidIndexNameException(expression.get(), "must not start with '_'."); } } - private static void ensureRemoteIndicesRequireIgnoreUnavailable(IndicesOptions options, List indexExpressions) { + private static void ensureRemoteIndicesRequireIgnoreUnavailable( + IndicesOptions options, + List resolvedExpressions + ) { if (options.ignoreUnavailable()) { return; } - for (String index : indexExpressions) { + for (ResolvedExpression resolvedExpression : resolvedExpressions) { + var index = resolvedExpression.resource(); if (RemoteClusterAware.isRemoteIndexName(index)) { - failOnRemoteIndicesNotIgnoringUnavailable(indexExpressions); + failOnRemoteIndicesNotIgnoringUnavailable(resolvedExpressions); } } } - private static void failOnRemoteIndicesNotIgnoringUnavailable(List indexExpressions) { + private static void failOnRemoteIndicesNotIgnoringUnavailable(List resolvedExpressions) { List crossClusterIndices = new ArrayList<>(); - for (String index : indexExpressions) { + for (ResolvedExpression resolvedExpression : resolvedExpressions) { + String index = resolvedExpression.resource(); if (RemoteClusterAware.isRemoteIndexName(index)) { crossClusterIndices.add(index); } @@ -1780,13 +1807,13 @@ public static final class ExpressionList implements Iterable expressionsList; private final boolean hasWildcard; - public record Expression(String expression, boolean isWildcard, boolean isExclusion) { + public record Expression(ResolvedExpression resolvedExpression, boolean isWildcard, boolean isExclusion) { public String get() { if (isExclusion()) { // drop the leading "-" if exclusion because it is easier for callers to handle it like this - return expression().substring(1); + return resolvedExpression().resource().substring(1); } else { - return expression(); + return resolvedExpression().resource(); } } } @@ -1795,16 +1822,17 @@ public String get() { * Creates the expression iterable that can be used to easily check which expression item is a wildcard or an exclusion (or both). * The {@param context} is used to check if wildcards ought to be considered or not. */ - public ExpressionList(Context context, List expressionStrings) { - List expressionsList = new ArrayList<>(expressionStrings.size()); + public ExpressionList(Context context, List resolvedExpressions) { + List expressionsList = new ArrayList<>(resolvedExpressions.size()); boolean wildcardSeen = false; - for (String expressionString : expressionStrings) { + for (ResolvedExpression resolvedExpression : resolvedExpressions) { + var expressionString = resolvedExpression.resource(); boolean isExclusion = expressionString.startsWith("-") && wildcardSeen; if (context.getOptions().expandWildcardExpressions() && isWildcard(expressionString)) { wildcardSeen = true; - expressionsList.add(new Expression(expressionString, true, isExclusion)); + expressionsList.add(new Expression(resolvedExpression, true, isExclusion)); } else { - expressionsList.add(new Expression(expressionString, false, isExclusion)); + expressionsList.add(new Expression(resolvedExpression, false, isExclusion)); } } this.expressionsList = expressionsList; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 0756080c16d00..b7777eca86179 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1316,23 +1316,6 @@ public Map templatesV2() { .orElse(Collections.emptyMap()); } - // TODO: remove this method: - public boolean isTimeSeriesTemplate(ComposableIndexTemplate indexTemplate) { - var indexModeFromTemplate = retrieveIndexModeFromTemplate(indexTemplate); - if (indexModeFromTemplate == IndexMode.TIME_SERIES) { - // No need to check for the existence of index.routing_path here, because index.mode=time_series can't be specified without it. - // Setting validation takes care of this. - // Also no need to validate that the fields defined in index.routing_path are keyword fields with time_series_dimension - // attribute enabled. This is validated elsewhere (DocumentMapper). - return true; - } - - // in a followup change: check the existence of keyword fields of type keyword and time_series_dimension attribute enabled in - // the template. In this case the index.routing_path setting can be generated from the mapping. - - return false; - } - public IndexMode retrieveIndexModeFromTemplate(ComposableIndexTemplate indexTemplate) { if (indexTemplate.getDataStreamTemplate() == null) { return null; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 1cebbabde0769..7f2c076281735 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -982,10 +982,10 @@ static Settings aggregateIndexSettings( if (sourceMetadata == null) { final Settings templateAndRequestSettings = Settings.builder().put(combinedTemplateSettings).put(request.settings()).build(); - final boolean timeSeriesTemplate = Optional.of(request) + final IndexMode templateIndexMode = Optional.of(request) .map(CreateIndexClusterStateUpdateRequest::matchingTemplate) - .map(metadata::isTimeSeriesTemplate) - .orElse(false); + .map(metadata::retrieveIndexModeFromTemplate) + .orElse(null); // Loop through all the explicit index setting providers, adding them to the // additionalIndexSettings map @@ -995,7 +995,7 @@ static Settings aggregateIndexSettings( var newAdditionalSettings = provider.getAdditionalIndexSettings( request.index(), request.dataStreamName(), - timeSeriesTemplate, + templateIndexMode, currentState.getMetadata(), resolvedAt, templateAndRequestSettings, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 8a46550f8a689..db3973c1a15a8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -45,6 +45,7 @@ public class MetadataDataStreamsService { private final DataStreamGlobalRetentionSettings globalRetentionSettings; private final MasterServiceTaskQueue updateLifecycleTaskQueue; private final MasterServiceTaskQueue setRolloverOnWriteTaskQueue; + private final MasterServiceTaskQueue updateOptionsTaskQueue; public MetadataDataStreamsService( ClusterService clusterService, @@ -93,6 +94,20 @@ public Tuple executeTask( Priority.NORMAL, rolloverOnWriteExecutor ); + ClusterStateTaskExecutor updateOptionsExecutor = new SimpleBatchedAckListenerTaskExecutor<>() { + + @Override + public Tuple executeTask( + UpdateOptionsTask modifyOptionsTask, + ClusterState clusterState + ) { + return new Tuple<>( + updateDataStreamOptions(clusterState, modifyOptionsTask.getDataStreamNames(), modifyOptionsTask.getOptions()), + modifyOptionsTask + ); + } + }; + this.updateOptionsTaskQueue = clusterService.createTaskQueue("modify-data-stream-options", Priority.NORMAL, updateOptionsExecutor); } public void modifyDataStream(final ModifyDataStreamsAction.Request request, final ActionListener listener) { @@ -147,6 +162,39 @@ public void removeLifecycle( ); } + /** + * Submits the task to set the provided data stream options to the requested data streams. + */ + public void setDataStreamOptions( + final List dataStreamNames, + DataStreamOptions options, + TimeValue ackTimeout, + TimeValue masterTimeout, + final ActionListener listener + ) { + updateOptionsTaskQueue.submitTask( + "set-data-stream-options", + new UpdateOptionsTask(dataStreamNames, options, ackTimeout, listener), + masterTimeout + ); + } + + /** + * Submits the task to remove the data stream options from the requested data streams. + */ + public void removeDataStreamOptions( + List dataStreamNames, + TimeValue ackTimeout, + TimeValue masterTimeout, + ActionListener listener + ) { + updateOptionsTaskQueue.submitTask( + "delete-data-stream-options", + new UpdateOptionsTask(dataStreamNames, null, ackTimeout, listener), + masterTimeout + ); + } + @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { clusterService.submitUnbatchedStateUpdateTask(source, task); @@ -228,6 +276,24 @@ ClusterState updateDataLifecycle(ClusterState currentState, List dataStr return ClusterState.builder(currentState).metadata(builder.build()).build(); } + /** + * Creates an updated cluster state in which the requested data streams have the data stream options provided. + * Visible for testing. + */ + ClusterState updateDataStreamOptions( + ClusterState currentState, + List dataStreamNames, + @Nullable DataStreamOptions dataStreamOptions + ) { + Metadata metadata = currentState.metadata(); + Metadata.Builder builder = Metadata.builder(metadata); + for (var dataStreamName : dataStreamNames) { + var dataStream = validateDataStream(metadata, dataStreamName); + builder.put(dataStream.copy().setDataStreamOptions(dataStreamOptions).build()); + } + return ClusterState.builder(currentState).metadata(builder.build()).build(); + } + /** * Creates an updated cluster state in which the requested data stream has the flag {@link DataStream#rolloverOnWrite()} * set to the value of the parameter rolloverOnWrite @@ -372,6 +438,34 @@ public DataStreamLifecycle getDataLifecycle() { } } + /** + * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion. + */ + static class UpdateOptionsTask extends AckedBatchedClusterStateUpdateTask { + + private final List dataStreamNames; + private final DataStreamOptions options; + + UpdateOptionsTask( + List dataStreamNames, + @Nullable DataStreamOptions options, + TimeValue ackTimeout, + ActionListener listener + ) { + super(ackTimeout, listener); + this.dataStreamNames = dataStreamNames; + this.options = options; + } + + public List getDataStreamNames() { + return dataStreamNames; + } + + public DataStreamOptions getOptions() { + return options; + } + } + /** * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion. */ diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 57194ded9422e..ccdfaa5518aee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -705,7 +705,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT var newAdditionalSettings = provider.getAdditionalIndexSettings( "validate-index-name", indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null, - indexTemplate.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(indexTemplate), + metadata.retrieveIndexModeFromTemplate(indexTemplate), currentState.getMetadata(), now, combinedSettings, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java index 4257543498c54..aa8b092ffcca0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java @@ -266,7 +266,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(NODE_ID_FIELD.getPreferredName(), nodeId); builder.field(TYPE_FIELD.getPreferredName(), type); builder.field(REASON_FIELD.getPreferredName(), reason); - builder.timeField(STARTED_AT_MILLIS_FIELD.getPreferredName(), STARTED_AT_READABLE_FIELD, startedAtMillis); + builder.timestampFieldsFromUnixEpochMillis( + STARTED_AT_MILLIS_FIELD.getPreferredName(), + STARTED_AT_READABLE_FIELD, + startedAtMillis + ); builder.field(NODE_SEEN_FIELD.getPreferredName(), nodeSeen); if (allocationDelay != null) { builder.field(ALLOCATION_DELAY_FIELD.getPreferredName(), allocationDelay.getStringRep()); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java index 3c559f9421a38..4c2f0cbaaf729 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; @@ -226,7 +227,7 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP public Settings getAdditionalIndexSettings( String indexName, @Nullable String dataStreamName, - boolean isTimeSeries, + IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index ad3d7d7f1c2ec..f5276bbe49b63 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.index.store.Store; @@ -186,6 +187,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, + IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, + SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java index 63859d89f3e22..db0b5b4357c7d 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java @@ -34,6 +34,27 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) { return new ChunkedToXContentBuilder(params); } + /** + * Create an iterator of {@link ToXContent} chunks for a REST response for the given {@link RestApiVersion}. Each chunk is serialized + * with the same {@link XContentBuilder} and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed + * as the {@code params} argument. For best results, all chunks should be {@code O(1)} size. The last chunk in the iterator must always + * yield at least one byte of output. See also {@link ChunkedToXContentHelper} for some handy utilities. + *

+ * Note that chunked response bodies cannot send deprecation warning headers once transmission has started, so implementations must + * check for deprecated feature use before returning. + *

+ * By default, delegates to {@link #toXContentChunked} or {#toXContentChunkedV8}. + * + * @return iterator over chunks of {@link ToXContent} + */ + default Iterator toXContentChunked(RestApiVersion restApiVersion, ToXContent.Params params) { + return switch (restApiVersion) { + case V_7 -> throw new AssertionError("v7 API not supported"); + case V_8 -> toXContentChunkedV8(params); + case V_9 -> toXContentChunked(params); + }; + } + /** * Create an iterator of {@link ToXContent} chunks for a REST response. Each chunk is serialized with the same {@link XContentBuilder} * and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed as the {@code params} argument. For @@ -48,12 +69,12 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) { Iterator toXContentChunked(ToXContent.Params params); /** - * Create an iterator of {@link ToXContent} chunks for a response to the {@link RestApiVersion#V_7} API. Each chunk is serialized with + * Create an iterator of {@link ToXContent} chunks for a response to the {@link RestApiVersion#V_8} API. Each chunk is serialized with * the same {@link XContentBuilder} and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed as the * {@code params} argument. For best results, all chunks should be {@code O(1)} size. The last chunk in the iterator must always yield * at least one byte of output. See also {@link ChunkedToXContentHelper} for some handy utilities. *

- * Similar to {@link #toXContentChunked} but for the {@link RestApiVersion#V_7} API. By default this method delegates to {@link + * Similar to {@link #toXContentChunked} but for the {@link RestApiVersion#V_8} API. By default this method delegates to {@link * #toXContentChunked}. *

* Note that chunked response bodies cannot send deprecation warning headers once transmission has started, so implementations must @@ -61,7 +82,7 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) { * * @return iterator over chunks of {@link ToXContent} */ - default Iterator toXContentChunkedV7(ToXContent.Params params) { + default Iterator toXContentChunkedV8(ToXContent.Params params) { return toXContentChunked(params); } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java index dea851b1b553a..0298e1a123b58 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java @@ -57,13 +57,13 @@ public Map, XContentBuilder.Writer> getXContentWriters() { // Fully-qualified here to reduce ambiguity around our (ES') Version class writers.put(org.apache.lucene.util.Version.class, (b, v) -> b.value(Objects.toString(v))); writers.put(TimeValue.class, (b, v) -> b.value(v.toString())); - writers.put(ZonedDateTime.class, XContentBuilder::timeValue); - writers.put(OffsetDateTime.class, XContentBuilder::timeValue); - writers.put(OffsetTime.class, XContentBuilder::timeValue); - writers.put(java.time.Instant.class, XContentBuilder::timeValue); - writers.put(LocalDateTime.class, XContentBuilder::timeValue); - writers.put(LocalDate.class, XContentBuilder::timeValue); - writers.put(LocalTime.class, XContentBuilder::timeValue); + writers.put(ZonedDateTime.class, XContentBuilder::timestampValue); + writers.put(OffsetDateTime.class, XContentBuilder::timestampValue); + writers.put(OffsetTime.class, XContentBuilder::timestampValue); + writers.put(java.time.Instant.class, XContentBuilder::timestampValue); + writers.put(LocalDateTime.class, XContentBuilder::timestampValue); + writers.put(LocalDate.class, XContentBuilder::timestampValue); + writers.put(LocalTime.class, XContentBuilder::timestampValue); writers.put(DayOfWeek.class, (b, v) -> b.value(v.toString())); writers.put(Month.class, (b, v) -> b.value(v.toString())); writers.put(MonthDay.class, (b, v) -> b.value(v.toString())); @@ -103,10 +103,8 @@ public Map, XContentBuilder.HumanReadableTransformer> getXContentHumanR public Map, Function> getDateTransformers() { Map, Function> transformers = new HashMap<>(); transformers.put(Date.class, d -> DEFAULT_FORMATTER.format(((Date) d).toInstant())); - transformers.put(Long.class, d -> DEFAULT_FORMATTER.format(Instant.ofEpochMilli((long) d))); transformers.put(Calendar.class, d -> DEFAULT_FORMATTER.format(((Calendar) d).toInstant())); transformers.put(GregorianCalendar.class, d -> DEFAULT_FORMATTER.format(((Calendar) d).toInstant())); - transformers.put(Instant.class, d -> DEFAULT_FORMATTER.format((Instant) d)); transformers.put(ZonedDateTime.class, d -> DEFAULT_FORMATTER.format((ZonedDateTime) d)); transformers.put(OffsetDateTime.class, d -> DEFAULT_FORMATTER.format((OffsetDateTime) d)); transformers.put(OffsetTime.class, d -> OFFSET_TIME_FORMATTER.format((OffsetTime) d)); @@ -119,4 +117,9 @@ public Map, Function> getDateTransformers() { transformers.put(LocalTime.class, d -> LOCAL_TIME_FORMATTER.format((LocalTime) d)); return transformers; } + + @Override + public String formatUnixEpochMillis(long unixEpochMillis) { + return DEFAULT_FORMATTER.format(Instant.ofEpochMilli(unixEpochMillis)); + } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index aaa4c738c0e13..0180d2c8df119 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -30,20 +30,21 @@ public interface IndexSettingProvider { * Returns explicitly set default index {@link Settings} for the given index. This should not * return null. * - * @param indexName The name of the new index being created - * @param dataStreamName The name of the data stream if the index being created is part of a data stream otherwise - * null - * @param isTimeSeries Whether the template is in time series mode. - * @param metadata The current metadata instance that doesn't yet contain the index to be created - * @param resolvedAt The time the request to create this new index was accepted. - * @param indexTemplateAndCreateRequestSettings All the settings resolved from the template that matches and any settings - * defined on the create index request - * @param combinedTemplateMappings All the mappings resolved from the template that matches + * @param indexName The name of the new index being created + * @param dataStreamName The name of the data stream if the index being created is part of a data stream + * otherwise null + * @param templateIndexMode The index mode defined in template if template creates data streams, + * otherwise null is returned. + * @param metadata The current metadata instance that doesn't yet contain the index to be created + * @param resolvedAt The time the request to create this new index was accepted. + * @param indexTemplateAndCreateRequestSettings All the settings resolved from the template that matches and any settings + * defined on the create index request + * @param combinedTemplateMappings All the mappings resolved from the template that matches */ Settings getAdditionalIndexSettings( String indexName, @Nullable String dataStreamName, - boolean isTimeSeries, + @Nullable IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index e82f9eff7d5e0..347b44a22e7c0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -28,6 +28,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.Node; @@ -651,6 +652,13 @@ public Iterator> settings() { Property.Final ); + public static final Setting SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting( + "index.synthetic_source.enable_second_doc_parsing_pass", + true, + Property.IndexScope, + Property.Dynamic + ); + /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -820,6 +828,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDimensionFieldsLimit; private volatile boolean skipIgnoredSourceWrite; private volatile boolean skipIgnoredSourceRead; + private volatile boolean syntheticSourceSecondDocParsingPassEnabled; + private final SourceFieldMapper.Mode indexMappingSourceMode; /** * The maximum number of refresh listeners allows on this shard. @@ -980,6 +990,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); + syntheticSourceSecondDocParsingPassEnabled = scopedSettings.get(SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); + indexMappingSourceMode = scopedSettings.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); scopedSettings.addSettingsUpdateConsumer( MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, @@ -1067,6 +1079,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this::setSkipIgnoredSourceWrite ); scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); + scopedSettings.addSettingsUpdateConsumer( + SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, + this::setSyntheticSourceSecondDocParsingPassEnabled + ); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1659,6 +1675,18 @@ private void setSkipIgnoredSourceRead(boolean value) { this.skipIgnoredSourceRead = value; } + private void setSyntheticSourceSecondDocParsingPassEnabled(boolean syntheticSourceSecondDocParsingPassEnabled) { + this.syntheticSourceSecondDocParsingPassEnabled = syntheticSourceSecondDocParsingPassEnabled; + } + + public boolean isSyntheticSourceSecondDocParsingPassEnabled() { + return syntheticSourceSecondDocParsingPassEnabled; + } + + public SourceFieldMapper.Mode getIndexMappingSourceMode() { + return indexMappingSourceMode; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java index 3d2acb533e26d..5201e57179cc7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java @@ -27,6 +27,14 @@ public class BQVectorUtils { private static final float EPSILON = 1e-4f; + public static double sqrtNewtonRaphson(double x, double curr, double prev) { + return (curr == prev) ? curr : sqrtNewtonRaphson(x, 0.5 * (curr + x / curr), curr); + } + + public static double constSqrt(double x) { + return x >= 0 && Double.isInfinite(x) == false ? sqrtNewtonRaphson(x, x, 0) : Double.NaN; + } + public static boolean isUnitVector(float[] v) { double l1norm = VectorUtil.dotProduct(v, v); return Math.abs(l1norm - 1.0d) <= EPSILON; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 4bf396e8d5ad1..10a20839ab3c5 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -41,6 +41,7 @@ import java.io.IOException; import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat { @@ -291,4 +292,9 @@ public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, Ra return delegate.getRandomVectorScorer(sim, values, query); } } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java index f0f25bd702749..7e586e210afd3 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java @@ -25,6 +25,8 @@ import java.io.IOException; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + class ES815BitFlatVectorsFormat extends FlatVectorsFormat { private static final FlatVectorsFormat delegate = new Lucene99FlatVectorsFormat(FlatBitVectorScorer.INSTANCE); @@ -43,6 +45,11 @@ public FlatVectorsReader fieldsReader(SegmentReadState segmentReadState) throws return delegate.fieldsReader(segmentReadState); } + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + static class FlatBitVectorScorer implements FlatVectorsScorer { static final FlatBitVectorScorer INSTANCE = new FlatBitVectorScorer(); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java index 78fa282709098..f4d22edc6dfdb 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java @@ -153,6 +153,7 @@ public static class BinarizedRandomVectorScorer extends RandomVectorScorer.Abstr private final VectorSimilarityFunction similarityFunction; private final float sqrtDimensions; + private final float maxX1; public BinarizedRandomVectorScorer( BinaryQueryVector queryVectors, @@ -164,24 +165,12 @@ public BinarizedRandomVectorScorer( this.targetVectors = targetVectors; this.similarityFunction = similarityFunction; // FIXME: precompute this once? - this.sqrtDimensions = (float) Utils.constSqrt(targetVectors.dimension()); - } - - // FIXME: utils class; pull this out - private static class Utils { - public static double sqrtNewtonRaphson(double x, double curr, double prev) { - return (curr == prev) ? curr : sqrtNewtonRaphson(x, 0.5 * (curr + x / curr), curr); - } - - public static double constSqrt(double x) { - return x >= 0 && Double.isInfinite(x) == false ? sqrtNewtonRaphson(x, x, 0) : Double.NaN; - } + this.sqrtDimensions = targetVectors.sqrtDimensions(); + this.maxX1 = targetVectors.maxX1(); } @Override public float score(int targetOrd) throws IOException { - // FIXME: implement fastscan in the future? - byte[] quantizedQuery = queryVector.vector(); int quantizedSum = queryVector.factors().quantizedSum(); float lower = queryVector.factors().lower(); @@ -218,17 +207,13 @@ public float score(int targetOrd) throws IOException { } assert Float.isFinite(dist); - // TODO: this is useful for mandatory rescoring by accounting for bias - // However, for just oversampling & rescoring, it isn't strictly useful. - // We should consider utilizing this bias in the future to determine which vectors need to - // be rescored - // float ooqSqr = (float) Math.pow(ooq, 2); - // float errorBound = (float) (normVmC * normOC * (maxX1 * Math.sqrt((1 - ooqSqr) / ooqSqr))); - // float score = dist - errorBound; + float ooqSqr = (float) Math.pow(ooq, 2); + float errorBound = (float) (vmC * normOC * (maxX1 * Math.sqrt((1 - ooqSqr) / ooqSqr))); + float score = Float.isFinite(errorBound) ? dist - errorBound : dist; if (similarityFunction == MAXIMUM_INNER_PRODUCT) { - return VectorUtil.scaleMaxInnerProductScore(dist); + return VectorUtil.scaleMaxInnerProductScore(score); } - return Math.max((1f + dist) / 2f, 0); + return Math.max((1f + score) / 2f, 0); } private float euclideanScore( @@ -256,17 +241,13 @@ private float euclideanScore( long qcDist = ESVectorUtil.ipByteBinByte(quantizedQuery, binaryCode); float score = sqrX + distanceToCentroid + factorPPC * lower + (qcDist * 2 - quantizedSum) * factorIP * width; - // TODO: this is useful for mandatory rescoring by accounting for bias - // However, for just oversampling & rescoring, it isn't strictly useful. - // We should consider utilizing this bias in the future to determine which vectors need to - // be rescored - // float projectionDist = (float) Math.sqrt(xX0 * xX0 - targetDistToC * targetDistToC); - // float error = 2.0f * maxX1 * projectionDist; - // float y = (float) Math.sqrt(distanceToCentroid); - // float errorBound = y * error; - // if (Float.isFinite(errorBound)) { - // score = dist + errorBound; - // } + float projectionDist = (float) Math.sqrt(xX0 * xX0 - targetDistToC * targetDistToC); + float error = 2.0f * maxX1 * projectionDist; + float y = (float) Math.sqrt(distanceToCentroid); + float errorBound = y * error; + if (Float.isFinite(errorBound)) { + score = score + errorBound; + } return Math.max(1 / (1f + score), 0); } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java index 2a3c3aca60e54..628480e273b34 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java @@ -34,6 +34,7 @@ import java.nio.ByteBuffer; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.elasticsearch.index.codec.vectors.BQVectorUtils.constSqrt; /** Binarized vector values loaded from off-heap */ public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorValues implements RandomAccessBinarizedByteVectorValues { @@ -53,6 +54,9 @@ public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorVa protected final BinaryQuantizer binaryQuantizer; protected final float[] centroid; protected final float centroidDp; + private final int discretizedDimensions; + private final float maxX1; + private final float sqrtDimensions; private final int correctionsCount; OffHeapBinarizedVectorValues( @@ -79,6 +83,9 @@ public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorVa this.byteBuffer = ByteBuffer.allocate(numBytes); this.binaryValue = byteBuffer.array(); this.binaryQuantizer = quantizer; + this.discretizedDimensions = BQVectorUtils.discretize(dimension, 64); + this.sqrtDimensions = (float) constSqrt(dimension); + this.maxX1 = (float) (1.9 / constSqrt(discretizedDimensions - 1.0)); } @Override @@ -103,6 +110,21 @@ public byte[] vectorValue(int targetOrd) throws IOException { return binaryValue; } + @Override + public int discretizedDimensions() { + return discretizedDimensions; + } + + @Override + public float sqrtDimensions() { + return sqrtDimensions; + } + + @Override + public float maxX1() { + return maxX1; + } + @Override public float getCentroidDP() { return centroidDp; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java index 2417353373ba5..5163baf617c29 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java @@ -24,6 +24,8 @@ import java.io.IOException; +import static org.elasticsearch.index.codec.vectors.BQVectorUtils.constSqrt; + /** * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 */ @@ -54,6 +56,18 @@ public interface RandomAccessBinarizedByteVectorValues extends RandomAccessVecto */ BinaryQuantizer getQuantizer(); + default int discretizedDimensions() { + return BQVectorUtils.discretize(dimension(), 64); + } + + default float sqrtDimensions() { + return (float) constSqrt(dimension()); + } + + default float maxX1() { + return (float) (1.9 / constSqrt(discretizedDimensions() - 1.0)); + } + /** * @return coarse grained centroids for the vectors */ diff --git a/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java index d321600e03bf9..90f8e6adab73d 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java +++ b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java @@ -15,11 +15,7 @@ import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.util.SameThreadExecutorService; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexSettings; @@ -29,8 +25,6 @@ import org.elasticsearch.index.shard.ShardId; import java.io.IOException; -import java.util.Collections; -import java.util.Locale; import java.util.Set; import java.util.concurrent.Executor; @@ -38,23 +32,13 @@ * An extension to the {@link ConcurrentMergeScheduler} that provides tracking on merge times, total * and current merges. */ -class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler { +public class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler implements ElasticsearchMergeScheduler { protected final Logger logger; private final Settings indexSettings; private final ShardId shardId; - private final MeanMetric totalMerges = new MeanMetric(); - private final CounterMetric totalMergesNumDocs = new CounterMetric(); - private final CounterMetric totalMergesSizeInBytes = new CounterMetric(); - private final CounterMetric currentMerges = new CounterMetric(); - private final CounterMetric currentMergesNumDocs = new CounterMetric(); - private final CounterMetric currentMergesSizeInBytes = new CounterMetric(); - private final CounterMetric totalMergeStoppedTime = new CounterMetric(); - private final CounterMetric totalMergeThrottledTime = new CounterMetric(); - - private final Set onGoingMerges = ConcurrentCollections.newConcurrentSet(); - private final Set readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges); + private final MergeTracking mergeTracking; private final MergeSchedulerConfig config; private final SameThreadExecutorService sameThreadExecutorService = new SameThreadExecutorService(); @@ -63,11 +47,16 @@ class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler { this.shardId = shardId; this.indexSettings = indexSettings.getSettings(); this.logger = Loggers.getLogger(getClass(), shardId); + this.mergeTracking = new MergeTracking( + logger, + () -> indexSettings.getMergeSchedulerConfig().isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY + ); refreshConfig(); } + @Override public Set onGoingMerges() { - return readOnlyOnGoingMerges; + return mergeTracking.onGoingMerges(); } /** We're currently only interested in messages with this prefix. */ @@ -104,74 +93,21 @@ protected void message(String message) { super.message(message); } - private static String getSegmentName(MergePolicy.OneMerge merge) { - return merge.getMergeInfo() != null ? merge.getMergeInfo().info.name : "_na_"; - } - @Override protected void doMerge(MergeSource mergeSource, MergePolicy.OneMerge merge) throws IOException { - int totalNumDocs = merge.totalNumDocs(); - long totalSizeInBytes = merge.totalBytesSize(); long timeNS = System.nanoTime(); - currentMerges.inc(); - currentMergesNumDocs.inc(totalNumDocs); - currentMergesSizeInBytes.inc(totalSizeInBytes); - OnGoingMerge onGoingMerge = new OnGoingMerge(merge); - onGoingMerges.add(onGoingMerge); - - if (logger.isTraceEnabled()) { - logger.trace( - "merge [{}] starting..., merging [{}] segments, [{}] docs, [{}] size, into [{}] estimated_size", - getSegmentName(merge), - merge.segments.size(), - totalNumDocs, - ByteSizeValue.ofBytes(totalSizeInBytes), - ByteSizeValue.ofBytes(merge.estimatedMergeBytes) - ); - } + mergeTracking.mergeStarted(onGoingMerge); try { beforeMerge(onGoingMerge); super.doMerge(mergeSource, merge); } finally { long tookMS = TimeValue.nsecToMSec(System.nanoTime() - timeNS); + mergeTracking.mergeFinished(merge, onGoingMerge, tookMS); - onGoingMerges.remove(onGoingMerge); afterMerge(onGoingMerge); - - currentMerges.dec(); - currentMergesNumDocs.dec(totalNumDocs); - currentMergesSizeInBytes.dec(totalSizeInBytes); - - totalMergesNumDocs.inc(totalNumDocs); - totalMergesSizeInBytes.inc(totalSizeInBytes); - totalMerges.inc(tookMS); - long stoppedMS = TimeValue.nsecToMSec( - merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.STOPPED) - ); - long throttledMS = TimeValue.nsecToMSec( - merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.PAUSED) - ); - totalMergeStoppedTime.inc(stoppedMS); - totalMergeThrottledTime.inc(throttledMS); - - String message = String.format( - Locale.ROOT, - "merge segment [%s] done: took [%s], [%,.1f MB], [%,d docs], [%s stopped], [%s throttled]", - getSegmentName(merge), - TimeValue.timeValueMillis(tookMS), - totalSizeInBytes / 1024f / 1024f, - totalNumDocs, - TimeValue.timeValueMillis(stoppedMS), - TimeValue.timeValueMillis(throttledMS) - ); - - if (tookMS > 20000) { // if more than 20 seconds, DEBUG log it - logger.debug("{}", message); - } else if (logger.isTraceEnabled()) { - logger.trace("{}", message); - } } + } /** @@ -206,24 +142,13 @@ protected MergeThread getMergeThread(MergeSource mergeSource, MergePolicy.OneMer return thread; } - MergeStats stats() { - final MergeStats mergeStats = new MergeStats(); - mergeStats.add( - totalMerges.count(), - totalMerges.sum(), - totalMergesNumDocs.count(), - totalMergesSizeInBytes.count(), - currentMerges.count(), - currentMergesNumDocs.count(), - currentMergesSizeInBytes.count(), - totalMergeStoppedTime.count(), - totalMergeThrottledTime.count(), - config.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY - ); - return mergeStats; + @Override + public MergeStats stats() { + return mergeTracking.stats(); } - void refreshConfig() { + @Override + public void refreshConfig() { if (this.getMaxMergeCount() != config.getMaxMergeCount() || this.getMaxThreadCount() != config.getMaxThreadCount()) { this.setMaxMergesAndThreads(config.getMaxMergeCount(), config.getMaxThreadCount()); } @@ -234,4 +159,9 @@ void refreshConfig() { disableAutoIOThrottle(); } } + + @Override + public MergeScheduler getMergeScheduler() { + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchMergeScheduler.java b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchMergeScheduler.java new file mode 100644 index 0000000000000..ac72c7a21da75 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchMergeScheduler.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.engine; + +import org.apache.lucene.index.MergeScheduler; +import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.merge.OnGoingMerge; + +import java.util.Set; + +public interface ElasticsearchMergeScheduler { + + Set onGoingMerges(); + + MergeStats stats(); + + void refreshConfig(); + + MergeScheduler getMergeScheduler(); +} diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index c72f5ce740d94..8d43252d178ee 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LiveIndexWriterConfig; import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; @@ -139,7 +140,7 @@ public class InternalEngine extends Engine { private volatile long lastDeleteVersionPruneTimeMSec; private final Translog translog; - private final ElasticsearchConcurrentMergeScheduler mergeScheduler; + private final ElasticsearchMergeScheduler mergeScheduler; private final IndexWriter indexWriter; @@ -248,11 +249,12 @@ public InternalEngine(EngineConfig engineConfig) { Translog translog = null; ExternalReaderManager externalReaderManager = null; ElasticsearchReaderManager internalReaderManager = null; - EngineMergeScheduler scheduler = null; + MergeScheduler scheduler = null; boolean success = false; try { this.lastDeleteVersionPruneTimeMSec = engineConfig.getThreadPool().relativeTimeInMillis(); - mergeScheduler = scheduler = new EngineMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings()); + mergeScheduler = createMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings()); + scheduler = mergeScheduler.getMergeScheduler(); throttle = new IndexThrottle(); try { store.trimUnsafeCommits(config().getTranslogConfig().getTranslogPath()); @@ -383,7 +385,7 @@ private SoftDeletesPolicy newSoftDeletesPolicy() throws IOException { @Nullable private CombinedDeletionPolicy.CommitsListener newCommitsListener() { - Engine.IndexCommitListener listener = engineConfig.getIndexCommitListener(); + IndexCommitListener listener = engineConfig.getIndexCommitListener(); if (listener != null) { final IndexCommitListener wrappedListener = Assertions.ENABLED ? assertingCommitsOrderListener(listener) : listener; return new CombinedDeletionPolicy.CommitsListener() { @@ -824,7 +826,7 @@ private GetResult getFromTranslog( config(), translogInMemorySegmentsCount::incrementAndGet ); - final Engine.Searcher searcher = new Engine.Searcher( + final Searcher searcher = new Searcher( "realtime_get", ElasticsearchDirectoryReader.wrap(inMemoryReader, shardId), config().getSimilarity(), @@ -841,7 +843,7 @@ public GetResult get( Get get, MappingLookup mappingLookup, DocumentParser documentParser, - Function searcherWrapper + Function searcherWrapper ) { try (var ignored = acquireEnsureOpenRef()) { if (get.realtime()) { @@ -875,7 +877,7 @@ protected GetResult realtimeGetUnderLock( Get get, MappingLookup mappingLookup, DocumentParser documentParser, - Function searcherWrapper, + Function searcherWrapper, boolean getFromSearcher ) { assert isDrainedForClose() == false; @@ -1098,7 +1100,7 @@ protected boolean assertPrimaryCanOptimizeAddDocument(final Index index) { return true; } - private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { + private boolean assertIncomingSequenceNumber(final Operation.Origin origin, final long seqNo) { if (origin == Operation.Origin.PRIMARY) { assert assertPrimaryIncomingSequenceNumber(origin, seqNo); } else { @@ -1108,7 +1110,7 @@ private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origi return true; } - protected boolean assertPrimaryIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { + protected boolean assertPrimaryIncomingSequenceNumber(final Operation.Origin origin, final long seqNo) { // sequence number should not be set when operation origin is primary assert seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : "primary operations must never have an assigned sequence number but was [" + seqNo + "]"; @@ -2700,7 +2702,7 @@ private IndexWriterConfig getIndexWriterConfig() { iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND); iwc.setIndexDeletionPolicy(combinedDeletionPolicy); iwc.setInfoStream(TESTS_VERBOSE ? InfoStream.getDefault() : new LoggerInfoStream(logger)); - iwc.setMergeScheduler(mergeScheduler); + iwc.setMergeScheduler(mergeScheduler.getMergeScheduler()); // Give us the opportunity to upgrade old segments while performing // background merges MergePolicy mergePolicy = config().getMergePolicy(); @@ -2753,7 +2755,7 @@ private IndexWriterConfig getIndexWriterConfig() { /** A listener that warms the segments if needed when acquiring a new reader */ static final class RefreshWarmerListener implements BiConsumer { - private final Engine.Warmer warmer; + private final Warmer warmer; private final Logger logger; private final AtomicBoolean isEngineClosed; @@ -2817,6 +2819,10 @@ LiveIndexWriterConfig getCurrentIndexWriterConfig() { return indexWriter.getConfig(); } + protected ElasticsearchMergeScheduler createMergeScheduler(ShardId shardId, IndexSettings indexSettings) { + return new EngineMergeScheduler(shardId, indexSettings); + } + private final class EngineMergeScheduler extends ElasticsearchConcurrentMergeScheduler { private final AtomicInteger numMergesInFlight = new AtomicInteger(0); private final AtomicBoolean isThrottling = new AtomicBoolean(); @@ -2827,7 +2833,7 @@ private final class EngineMergeScheduler extends ElasticsearchConcurrentMergeSch @Override public synchronized void beforeMerge(OnGoingMerge merge) { - int maxNumMerges = mergeScheduler.getMaxMergeCount(); + int maxNumMerges = getMaxMergeCount(); if (numMergesInFlight.incrementAndGet() > maxNumMerges) { if (isThrottling.getAndSet(true) == false) { logger.info("now throttling indexing: numMergesInFlight={}, maxNumMerges={}", numMergesInFlight, maxNumMerges); @@ -2838,7 +2844,7 @@ public synchronized void beforeMerge(OnGoingMerge merge) { @Override public synchronized void afterMerge(OnGoingMerge merge) { - int maxNumMerges = mergeScheduler.getMaxMergeCount(); + int maxNumMerges = getMaxMergeCount(); if (numMergesInFlight.decrementAndGet() < maxNumMerges) { if (isThrottling.getAndSet(false)) { logger.info("stop throttling indexing: numMergesInFlight={}, maxNumMerges={}", numMergesInFlight, maxNumMerges); @@ -2876,25 +2882,29 @@ protected void doRun() { @Override protected void handleMergeException(final Throwable exc) { - engineConfig.getThreadPool().generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - logger.debug("merge failure action rejected", e); - } - - @Override - protected void doRun() throws Exception { - /* - * We do this on another thread rather than the merge thread that we are initially called on so that we have complete - * confidence that the call stack does not contain catch statements that would cause the error that might be thrown - * here from being caught and never reaching the uncaught exception handler. - */ - failEngine("merge failed", new MergePolicy.MergeException(exc)); - } - }); + mergeException(exc); } } + protected void mergeException(final Throwable exc) { + engineConfig.getThreadPool().generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + logger.debug("merge failure action rejected", e); + } + + @Override + protected void doRun() throws Exception { + /* + * We do this on another thread rather than the merge thread that we are initially called on so that we have complete + * confidence that the call stack does not contain catch statements that would cause the error that might be thrown + * here from being caught and never reaching the uncaught exception handler. + */ + failEngine("merge failed", new MergePolicy.MergeException(exc)); + } + }); + } + /** * Commits the specified index writer. * diff --git a/server/src/main/java/org/elasticsearch/index/engine/MergeTracking.java b/server/src/main/java/org/elasticsearch/index/engine/MergeTracking.java new file mode 100644 index 0000000000000..3f52b607cf356 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/engine/MergeTracking.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.engine; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.MergePolicy; +import org.elasticsearch.common.metrics.CounterMetric; +import org.elasticsearch.common.metrics.MeanMetric; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.merge.OnGoingMerge; + +import java.util.Collections; +import java.util.Locale; +import java.util.Set; +import java.util.function.DoubleSupplier; + +public class MergeTracking { + + protected final Logger logger; + private final DoubleSupplier mbPerSecAutoThrottle; + + private final MeanMetric totalMerges = new MeanMetric(); + private final CounterMetric totalMergesNumDocs = new CounterMetric(); + private final CounterMetric totalMergesSizeInBytes = new CounterMetric(); + private final CounterMetric currentMerges = new CounterMetric(); + private final CounterMetric currentMergesNumDocs = new CounterMetric(); + private final CounterMetric currentMergesSizeInBytes = new CounterMetric(); + private final CounterMetric totalMergeStoppedTime = new CounterMetric(); + private final CounterMetric totalMergeThrottledTime = new CounterMetric(); + + private final Set onGoingMerges = ConcurrentCollections.newConcurrentSet(); + private final Set readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges); + + public MergeTracking(Logger logger, DoubleSupplier mbPerSecAutoThrottle) { + this.logger = logger; + this.mbPerSecAutoThrottle = mbPerSecAutoThrottle; + } + + public Set onGoingMerges() { + return readOnlyOnGoingMerges; + } + + public void mergeStarted(OnGoingMerge onGoingMerge) { + MergePolicy.OneMerge merge = onGoingMerge.getMerge(); + int totalNumDocs = merge.totalNumDocs(); + long totalSizeInBytes = merge.totalBytesSize(); + currentMerges.inc(); + currentMergesNumDocs.inc(totalNumDocs); + currentMergesSizeInBytes.inc(totalSizeInBytes); + onGoingMerges.add(onGoingMerge); + + if (logger.isTraceEnabled()) { + logger.trace( + "merge [{}] starting: merging [{}] segments, [{}] docs, [{}] size, into [{}] estimated_size", + onGoingMerge.getId(), + merge.segments.size(), + totalNumDocs, + ByteSizeValue.ofBytes(totalSizeInBytes), + ByteSizeValue.ofBytes(merge.estimatedMergeBytes) + ); + } + } + + public void mergeFinished(final MergePolicy.OneMerge merge, final OnGoingMerge onGoingMerge, long tookMS) { + int totalNumDocs = merge.totalNumDocs(); + long totalSizeInBytes = merge.totalBytesSize(); + + onGoingMerges.remove(onGoingMerge); + + currentMerges.dec(); + currentMergesNumDocs.dec(totalNumDocs); + currentMergesSizeInBytes.dec(totalSizeInBytes); + + totalMergesNumDocs.inc(totalNumDocs); + totalMergesSizeInBytes.inc(totalSizeInBytes); + totalMerges.inc(tookMS); + long stoppedMS = TimeValue.nsecToMSec( + merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.STOPPED) + ); + long throttledMS = TimeValue.nsecToMSec( + merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.PAUSED) + ); + totalMergeStoppedTime.inc(stoppedMS); + totalMergeThrottledTime.inc(throttledMS); + + String message = String.format( + Locale.ROOT, + "merge [%s] segment [%s] done: took [%s], [%s], [%,d] docs, [%s] stopped, [%s] throttled", + onGoingMerge.getId(), + getSegmentName(merge), + TimeValue.timeValueMillis(tookMS), + ByteSizeValue.ofBytes(totalSizeInBytes), + totalNumDocs, + TimeValue.timeValueMillis(stoppedMS), + TimeValue.timeValueMillis(throttledMS) + ); + + if (tookMS > 20000) { // if more than 20 seconds, DEBUG log it + logger.debug("{}", message); + } else if (logger.isTraceEnabled()) { + logger.trace("{}", message); + } + } + + public MergeStats stats() { + final MergeStats mergeStats = new MergeStats(); + mergeStats.add( + totalMerges.count(), + totalMerges.sum(), + totalMergesNumDocs.count(), + totalMergesSizeInBytes.count(), + currentMerges.count(), + currentMergesNumDocs.count(), + currentMergesSizeInBytes.count(), + totalMergeStoppedTime.count(), + totalMergeThrottledTime.count(), + mbPerSecAutoThrottle.getAsDouble() + ); + return mergeStats; + } + + private static String getSegmentName(MergePolicy.OneMerge merge) { + return merge.getMergeInfo() != null ? merge.getMergeInfo().info.name : "_na_"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index ac236e5a7e5fd..2eec14bd1a8d6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -111,6 +111,7 @@ public int get() { private final Set ignoredFields; private final List ignoredFieldValues; private final List ignoredFieldsMissingValues; + private final boolean inArrayScopeEnabled; private boolean inArrayScope; private final Map> dynamicMappers; @@ -143,6 +144,7 @@ private DocumentParserContext( Set ignoreFields, List ignoredFieldValues, List ignoredFieldsWithNoSource, + boolean inArrayScopeEnabled, boolean inArrayScope, Map> dynamicMappers, Map dynamicObjectMappers, @@ -164,6 +166,7 @@ private DocumentParserContext( this.ignoredFields = ignoreFields; this.ignoredFieldValues = ignoredFieldValues; this.ignoredFieldsMissingValues = ignoredFieldsWithNoSource; + this.inArrayScopeEnabled = inArrayScopeEnabled; this.inArrayScope = inArrayScope; this.dynamicMappers = dynamicMappers; this.dynamicObjectMappers = dynamicObjectMappers; @@ -188,6 +191,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.ignoredFields, in.ignoredFieldValues, in.ignoredFieldsMissingValues, + in.inArrayScopeEnabled, in.inArrayScope, in.dynamicMappers, in.dynamicObjectMappers, @@ -219,6 +223,7 @@ protected DocumentParserContext( new HashSet<>(), new ArrayList<>(), new ArrayList<>(), + mappingParserContext.getIndexSettings().isSyntheticSourceSecondDocParsingPassEnabled(), false, new HashMap<>(), new HashMap<>(), @@ -371,7 +376,7 @@ public final Collection getIgnoredFieldsMiss * Applies to synthetic source only. */ public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { - if (canAddIgnoredField() && mapper instanceof ObjectMapper) { + if (canAddIgnoredField() && mapper instanceof ObjectMapper && inArrayScopeEnabled) { boolean isNested = mapper instanceof NestedObjectMapper; if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { DocumentParserContext subcontext = switchParser(parser()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index f3744c974e9e3..dbaa1f3a04ab9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettings; @@ -28,7 +29,7 @@ public class MapperFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of( + Set features = Set.of( BWC_WORKAROUND_9_0, IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, PassThroughObjectMapper.PASS_THROUGH_PRIORITY, @@ -54,6 +55,11 @@ public Set getFeatures() { TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF, FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT ); + // BBQ is currently behind a feature flag for testing + if (DenseVectorFieldMapper.BBQ_FEATURE_FLAG.isEnabled()) { + return Sets.union(features, Set.of(DenseVectorFieldMapper.BBQ_FORMAT)); + } + return features; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 5e63fee8c5adc..70c4a3ac213a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; @@ -41,6 +42,7 @@ public class ObjectMapper extends Mapper { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class); + public static final FeatureFlag SUB_OBJECTS_AUTO_FEATURE_FLAG = new FeatureFlag("sub_objects_auto"); public static final String CONTENT_TYPE = "object"; static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source"; @@ -74,7 +76,7 @@ public static Subobjects from(Object node) { if (value.equalsIgnoreCase("false")) { return DISABLED; } - if (value.equalsIgnoreCase("auto")) { + if (SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled() && value.equalsIgnoreCase("auto")) { return AUTO; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index 80f845d626a2f..decc6d40a2f8e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -34,9 +34,6 @@ * In case different pass-through objects contain subfields with the same name (excluding the pass-through prefix), their aliases conflict. * To resolve this, the pass-through spec specifies which object takes precedence through required parameter "priority"; non-negative * integer values are accepted, with the highest priority value winning in case of conflicting aliases. - * - * Note that this is an experimental, undocumented mapper type, currently intended for prototyping purposes only. - * It has not been vetted for use in production systems. */ public class PassThroughObjectMapper extends ObjectMapper { public static final String CONTENT_TYPE = "passthrough"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 0f4549c679d42..f9b9de97715ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,11 +18,13 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; @@ -62,8 +64,16 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; + public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { + final IndexMode indexMode = IndexSettings.MODE.get(settings); + return switch (indexMode) { + case IndexMode.LOGSDB, IndexMode.TIME_SERIES -> Mode.SYNTHETIC.name(); + default -> Mode.STORED.name(); + }; + }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); + /** The source mode */ - private enum Mode { + public enum Mode { DISABLED, STORED, SYNTHETIC @@ -96,6 +106,15 @@ private enum Mode { true ); + private static final SourceFieldMapper TSDB_DEFAULT_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + true + ); + private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, @@ -105,6 +124,15 @@ private enum Mode { false ); + private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + false + ); + private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, @@ -114,6 +142,15 @@ private enum Mode { true ); + private static final SourceFieldMapper LOGSDB_DEFAULT_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.LOGSDB, + true + ); + private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, @@ -123,6 +160,15 @@ private enum Mode { false ); + private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.LOGSDB, + false + ); + /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. @@ -197,6 +243,8 @@ public static class Builder extends MetadataFieldMapper.Builder { m -> Arrays.asList(toType(m).excludes) ); + private final Settings settings; + private final IndexMode indexMode; private final boolean supportsNonDefaultParameterValues; @@ -210,6 +258,7 @@ public Builder( boolean enableRecoverySource ) { super(Defaults.NAME); + this.settings = settings; this.indexMode = indexMode; this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false || settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true); @@ -226,10 +275,10 @@ protected Parameter[] getParameters() { return new Parameter[] { enabled, mode, includes, excludes }; } - private boolean isDefault() { - Mode m = mode.get(); - if (m != null - && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) { + private boolean isDefault(final Mode sourceMode) { + if (sourceMode != null + && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && sourceMode == Mode.SYNTHETIC) == false) + || sourceMode == Mode.DISABLED)) { return false; } return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty(); @@ -242,12 +291,14 @@ public SourceFieldMapper build() { throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters"); } } - if (isDefault()) { - return switch (indexMode) { - case TIME_SERIES -> enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; - case LOGSDB -> enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; - default -> enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; - }; + // NOTE: if the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source` + // otherwise the mode is determined according to `index.mode` and `_source.mode`. + final Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings) + ? INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings) + : mode.get(); + if (isDefault(sourceMode)) { + return resolveSourceMode(indexMode, sourceMode, enableRecoverySource); + } if (supportsNonDefaultParameterValues == false) { List disallowed = new ArrayList<>(); @@ -271,8 +322,9 @@ public SourceFieldMapper build() { ); } } + SourceFieldMapper sourceFieldMapper = new SourceFieldMapper( - mode.get(), + sourceMode, enabled.get(), includes.getValue().toArray(Strings.EMPTY_ARRAY), excludes.getValue().toArray(Strings.EMPTY_ARRAY), @@ -287,21 +339,39 @@ public SourceFieldMapper build() { } + private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, final Mode sourceMode, boolean enableRecoverySource) { + if (indexMode == IndexMode.STANDARD) { + return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; + } + final SourceFieldMapper syntheticWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES + ? TSDB_DEFAULT_NO_RECOVERY_SOURCE + : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; + final SourceFieldMapper syntheticWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT; + final SourceFieldMapper storedWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES + ? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED + : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED; + final SourceFieldMapper storedWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED; + + return switch (sourceMode) { + case SYNTHETIC -> enableRecoverySource ? syntheticWithRecoverySource : syntheticWithoutRecoverySource; + case STORED -> enableRecoverySource ? storedWithRecoverySource : storedWithoutRecoverySource; + case DISABLED -> throw new IllegalArgumentException( + "_source can not be disabled in index using [" + indexMode + "] index mode" + ); + }; + } + public static final TypeParser PARSER = new ConfigurableTypeParser(c -> { - var indexMode = c.getIndexSettings().getMode(); + final IndexMode indexMode = c.getIndexSettings().getMode(); boolean enableRecoverySource = INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings()); + final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); + if (indexMode.isSyntheticSourceEnabled()) { - if (indexMode == IndexMode.TIME_SERIES) { - if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) { - return enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; - } else { - return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE; - } - } else if (indexMode == IndexMode.LOGSDB) { - return enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; + if (indexMode == IndexMode.TIME_SERIES && c.getIndexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)) { + return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE; } } - return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; + return resolveSourceMode(indexMode, settingSourceMode, enableRecoverySource); }, c -> new Builder( c.getIndexSettings().getMode(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index d7353584706d8..52ff7a3014d1d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -36,6 +36,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.VectorUtil; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; @@ -45,6 +46,8 @@ import org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat; import org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat; import org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat; +import org.elasticsearch.index.codec.vectors.ES816BinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.ES816HnswBinaryQuantizedVectorsFormat; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.ArraySourceValueFetcher; @@ -98,6 +101,7 @@ public class DenseVectorFieldMapper extends FieldMapper { public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude"; private static final float EPS = 1e-3f; + static final int BBQ_MIN_DIMS = 64; public static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; @@ -105,6 +109,8 @@ public static boolean isNotUnitVector(float magnitude) { public static final NodeFeature INT4_QUANTIZATION = new NodeFeature("mapper.vectors.int4_quantization"); public static final NodeFeature BIT_VECTORS = new NodeFeature("mapper.vectors.bit_vectors"); + public static final NodeFeature BBQ_FORMAT = new NodeFeature("mapper.vectors.bbq"); + public static final FeatureFlag BBQ_FEATURE_FLAG = new FeatureFlag("bbq_index_format"); public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersions.V_7_5_0; public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION; @@ -1162,7 +1168,7 @@ final void validateElementType(ElementType elementType) { abstract boolean updatableTo(IndexOptions update); - public final void validateDimension(int dim) { + public void validateDimension(int dim) { if (type.supportsDimension(dim)) { return; } @@ -1342,6 +1348,50 @@ public boolean supportsElementType(ElementType elementType) { public boolean supportsDimension(int dims) { return dims % 2 == 0; } + }, + BBQ_HNSW("bbq_hnsw") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + Object mNode = indexOptionsMap.remove("m"); + Object efConstructionNode = indexOptionsMap.remove("ef_construction"); + if (mNode == null) { + mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + } + if (efConstructionNode == null) { + efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + } + int m = XContentMapValues.nodeIntegerValue(mNode); + int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new BBQHnswIndexOptions(m, efConstruction); + } + + @Override + public boolean supportsElementType(ElementType elementType) { + return elementType == ElementType.FLOAT; + } + + @Override + public boolean supportsDimension(int dims) { + return dims >= BBQ_MIN_DIMS; + } + }, + BBQ_FLAT("bbq_flat") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new BBQFlatIndexOptions(); + } + + @Override + public boolean supportsElementType(ElementType elementType) { + return elementType == ElementType.FLOAT; + } + + @Override + public boolean supportsDimension(int dims) { + return dims >= BBQ_MIN_DIMS; + } }; static Optional fromString(String type) { @@ -1707,6 +1757,102 @@ public String toString() { } } + static class BBQHnswIndexOptions extends IndexOptions { + private final int m; + private final int efConstruction; + + BBQHnswIndexOptions(int m, int efConstruction) { + super(VectorIndexType.BBQ_HNSW); + this.m = m; + this.efConstruction = efConstruction; + } + + @Override + KnnVectorsFormat getVectorsFormat(ElementType elementType) { + assert elementType == ElementType.FLOAT; + return new ES816HnswBinaryQuantizedVectorsFormat(m, efConstruction); + } + + @Override + boolean updatableTo(IndexOptions update) { + return update.type.equals(this.type); + } + + @Override + boolean doEquals(IndexOptions other) { + BBQHnswIndexOptions that = (BBQHnswIndexOptions) other; + return m == that.m && efConstruction == that.efConstruction; + } + + @Override + int doHashCode() { + return Objects.hash(m, efConstruction); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.field("m", m); + builder.field("ef_construction", efConstruction); + builder.endObject(); + return builder; + } + + @Override + public void validateDimension(int dim) { + if (type.supportsDimension(dim)) { + return; + } + throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim); + } + } + + static class BBQFlatIndexOptions extends IndexOptions { + private final int CLASS_NAME_HASH = this.getClass().getName().hashCode(); + + BBQFlatIndexOptions() { + super(VectorIndexType.BBQ_FLAT); + } + + @Override + KnnVectorsFormat getVectorsFormat(ElementType elementType) { + assert elementType == ElementType.FLOAT; + return new ES816BinaryQuantizedVectorsFormat(); + } + + @Override + boolean updatableTo(IndexOptions update) { + return update.type.equals(this.type); + } + + @Override + boolean doEquals(IndexOptions other) { + return other instanceof BBQFlatIndexOptions; + } + + @Override + int doHashCode() { + return CLASS_NAME_HASH; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.endObject(); + return builder; + } + + @Override + public void validateDimension(int dim) { + if (type.supportsDimension(dim)) { + return; + } + throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim); + } + } + public static final TypeParser PARSER = new TypeParser( (n, c) -> new Builder(n, c.indexVersionCreated()), notInMultiFields(CONTENT_TYPE) @@ -2108,9 +2254,15 @@ private static IndexOptions parseIndexOptions(String fieldName, Object propNode) throw new MapperParsingException("[index_options] requires field [type] to be configured"); } String type = XContentMapValues.nodeStringValue(typeNode); - return VectorIndexType.fromString(type) - .orElseThrow(() -> new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]")) - .parseIndexOptions(fieldName, indexOptionsMap); + Optional vectorIndexType = VectorIndexType.fromString(type); + if (vectorIndexType.isEmpty()) { + throw new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]"); + } + VectorIndexType parsedType = vectorIndexType.get(); + if ((parsedType == VectorIndexType.BBQ_FLAT || parsedType == VectorIndexType.BBQ_HNSW) && BBQ_FEATURE_FLAG.isEnabled() == false) { + throw new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]"); + } + return parsedType.parseIndexOptions(fieldName, indexOptionsMap); } /** @@ -2270,7 +2422,7 @@ public void write(XContentBuilder b) throws IOException { if (indexCreatedVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION)) { byteBuffer.order(ByteOrder.LITTLE_ENDIAN); } - int dims = fieldType().dims; + int dims = fieldType().elementType == ElementType.BIT ? fieldType().dims / Byte.SIZE : fieldType().dims; for (int dim = 0; dim < dims; dim++) { fieldType().elementType.readAndWriteValue(byteBuffer, b); } diff --git a/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java b/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java index df49e00f8af73..7c40fdc93a48b 100644 --- a/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java +++ b/server/src/main/java/org/elasticsearch/index/merge/OnGoingMerge.java @@ -50,4 +50,8 @@ public long getTotalBytesSize() { public List getMergedSegments() { return oneMerge.segments; } + + public MergePolicy.OneMerge getMerge() { + return oneMerge; + } } diff --git a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java index 186aff230b8d0..387385ea2d6a4 100644 --- a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java +++ b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java @@ -81,7 +81,9 @@ public enum LuceneFilesExtensions { VEM("vem", "Vector Metadata", true, false), VEMF("vemf", "Flat Vector Metadata", true, false), VEMQ("vemq", "Scalar Quantized Vector Metadata", true, false), - VEQ("veq", "Scalar Quantized Vector Data", false, true); + VEQ("veq", "Scalar Quantized Vector Data", false, true), + VEMB("vemb", "Binarized Vector Metadata", true, false), + VEB("veb", "Binarized Vector Data", false, true); /** * Allow plugin developers of custom codecs to opt out of the assertion in {@link #fromExtension} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 706f788e8a310..2dc5e7c28ad0b 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; @@ -1713,7 +1714,7 @@ interface IndexDeletionAllowedPredicate { IndexSettings indexSettings) -> canDeleteIndexContents(index); private final IndexDeletionAllowedPredicate ALWAYS_TRUE = (Index index, IndexSettings indexSettings) -> true; - public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { + public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { /* Being static, parseAliasFilter doesn't have access to whatever guts it needs to parse a query. Instead of passing in a bunch * of dependencies we pass in a function that can perform the parsing. */ CheckedFunction filterParser = bytes -> { diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java index 6be94ab21a4f7..b0d33a75ba883 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java @@ -293,9 +293,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.TYPE, recoverySource.getType()); builder.field(Fields.STAGE, stage.toString()); builder.field(Fields.PRIMARY, primary); - builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, timer.startTime); + builder.timestampFieldsFromUnixEpochMillis(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, timer.startTime); if (timer.stopTime > 0) { - builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, timer.stopTime); + builder.timestampFieldsFromUnixEpochMillis(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, timer.stopTime); } builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(timer.time())); diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index cbbfef2cc65fa..835262ff28edc 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -192,12 +192,26 @@ default boolean canStream(TaskType taskType) { return supportedStreamingTasks().contains(taskType); } + record DefaultConfigId(String inferenceId, TaskType taskType, InferenceService service) {}; + /** - * A service can define default configurations that can be - * used out of the box without creating an endpoint first. - * @return Default configurations provided by this service + * Get the Ids and task type of any default configurations provided by this service + * @return Defaults */ - default List defaultConfigs() { + default List defaultConfigIds() { return List.of(); } + + /** + * Call the listener with the default model configurations defined by + * the service + * @param defaultsListener The listener + */ + default void defaultConfigs(ActionListener> defaultsListener) { + defaultsListener.onResponse(List.of()); + } + + default void updateModelsWithDynamicFields(List model, ActionListener> listener) { + listener.onResponse(model); + } } diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java index 68dc865b4c7db..3274bf571d10a 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceExtension.java @@ -10,6 +10,8 @@ package org.elasticsearch.inference; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; import java.util.List; @@ -21,7 +23,7 @@ public interface InferenceServiceExtension { List getInferenceServiceFactories(); - record InferenceServiceFactoryContext(Client client, ThreadPool threadPool) {} + record InferenceServiceFactoryContext(Client client, ThreadPool threadPool, ClusterService clusterService, Settings settings) {} interface Factory { /** diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index 1c68615203d3a..a3639214a1b9d 100644 --- a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -420,7 +420,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.VM_VERSION, vmVersion); builder.field(Fields.VM_VENDOR, vmVendor); builder.field(Fields.USING_BUNDLED_JDK, usingBundledJdk); - builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, startTime); + builder.timestampFieldsFromUnixEpochMillis(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, startTime); builder.startObject(Fields.MEM); builder.humanReadableField(Fields.HEAP_INIT_IN_BYTES, Fields.HEAP_INIT, ByteSizeValue.ofBytes(mem.heapInit)); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 5024cc5468866..32a65302922a8 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -561,7 +561,7 @@ public synchronized void close() throws IOException { toClose.add(() -> stopWatch.stop().start("transport")); toClose.add(injector.getInstance(TransportService.class)); toClose.add(injector.getInstance(NodeMetrics.class)); - toClose.add(injector.getInstance(IndicesService.class)); + toClose.add(injector.getInstance(IndicesMetrics.class)); if (ReadinessService.enabled(environment)) { toClose.add(injector.getInstance(ReadinessService.class)); } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index fc39d2d2d80a4..2b95fbc69199f 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -937,8 +937,7 @@ private static boolean indexSettingsMatchRepositoryMetadata(IndexMetadata indexM private static RepositoryConflictException newRepositoryConflictException(String repository, String reason) { return new RepositoryConflictException( repository, - "trying to modify or unregister repository that is currently used (" + reason + ')', - "trying to modify or unregister repository [" + repository + "] that is currently used (" + reason + ')' + "trying to modify or unregister repository that is currently used (" + reason + ')' ); } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java index ee4ffa86a2475..15a6b0d3791d8 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryConflictException.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -19,11 +20,8 @@ * Repository conflict exception */ public class RepositoryConflictException extends RepositoryException { - private final String backwardCompatibleMessage; - - public RepositoryConflictException(String repository, String message, String backwardCompatibleMessage) { + public RepositoryConflictException(String repository, String message) { super(repository, message); - this.backwardCompatibleMessage = backwardCompatibleMessage; } @Override @@ -31,18 +29,16 @@ public RestStatus status() { return RestStatus.CONFLICT; } - public String getBackwardCompatibleMessage() { - return backwardCompatibleMessage; - } - + @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // drop unneeded string from wire format public RepositoryConflictException(StreamInput in) throws IOException { super(in); - this.backwardCompatibleMessage = in.readString(); + in.readString(); } @Override + @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // drop unneeded string from wire format protected void writeTo(StreamOutput out, Writer nestedExceptionsWriter) throws IOException { super.writeTo(out, nestedExceptionsWriter); - out.writeString(backwardCompatibleMessage); + out.writeString(""); } } diff --git a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java index b28d7c26b11bb..694af7e1606cb 100644 --- a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java +++ b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBodyPart.java @@ -19,7 +19,6 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Releasables; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Streams; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -129,9 +128,10 @@ public void write(byte[] b, int off, int len) throws IOException { Streams.noCloseStream(out) ); - private final Iterator serialization = builder.getRestApiVersion() == RestApiVersion.V_7 - ? chunkedToXContent.toXContentChunkedV7(params) - : chunkedToXContent.toXContentChunked(params); + private final Iterator serialization = chunkedToXContent.toXContentChunked( + builder.getRestApiVersion(), + params + ); private BytesStream target; diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index 0e3b8d37dd25c..cf66e402d3691 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -197,16 +197,6 @@ private RouteBuilder(Method method, String path) { this.restApiVersion = RestApiVersion.current(); } - /** - * @deprecated Use {@link #deprecatedForRemoval(String, RestApiVersion)} if the intent is deprecate the path and remove in the - * next major version. Use {@link #deprecateAndKeep(String)} if the intent is to deprecate the path but not remove it. - * This method will delegate to {@link #deprecatedForRemoval(String, RestApiVersion)}. - */ - @Deprecated(since = "9.0.0", forRemoval = true) - public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { - return deprecatedForRemoval(deprecationMessage, lastFullySupportedVersion); - } - /** * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage) for removal. Notes the last * major version in which the path is fully supported without compatibility headers. If this path is being replaced by another diff --git a/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java b/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java index 7f61d171fae33..db33673939ae9 100644 --- a/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java +++ b/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java @@ -25,7 +25,6 @@ import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Streams; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.ToXContent; @@ -125,9 +124,7 @@ public void close() { } private Iterator getChunksIterator(StreamingFragment fragment) { - return xContentBuilder.getRestApiVersion() == RestApiVersion.V_7 - ? fragment.fragment().toXContentChunkedV7(params) - : fragment.fragment().toXContentChunked(params); + return fragment.fragment().toXContentChunked(xContentBuilder.getRestApiVersion(), params); } /** diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java index c8eb80cdcfdd7..e66b7d2b0b1a4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -28,10 +27,6 @@ public class RestAddVotingConfigExclusionAction extends BaseRestHandler { private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(30L); - private static final String DEPRECATION_MESSAGE = "POST /_cluster/voting_config_exclusions/{node_name} " - + "has been removed. You must use POST /_cluster/voting_config_exclusions?node_ids=... " - + "or POST /_cluster/voting_config_exclusions?node_names=... instead."; - @Override public String getName() { return "add_voting_config_exclusions_action"; @@ -39,12 +34,7 @@ public String getName() { @Override public List routes() { - return List.of( - new Route(POST, "/_cluster/voting_config_exclusions"), - Route.builder(POST, "/_cluster/voting_config_exclusions/{node_name}") - .deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7) - .build() - ); + return List.of(new Route(POST, "/_cluster/voting_config_exclusions")); } @Override @@ -66,10 +56,6 @@ static AddVotingConfigExclusionsRequest resolveVotingConfigExclusionsRequest(fin String nodeIds = null; String nodeNames = null; - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("node_name")) { - throw new IllegalArgumentException("[node_name] has been removed, you must set [node_names] or [node_ids]"); - } - if (request.hasParam("node_ids")) { nodeIds = request.param("node_ids"); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java index d2f5ca31ae3d1..c512db85d3023 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java @@ -10,10 +10,7 @@ package org.elasticsearch.rest.action.admin.cluster; import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.repositories.RepositoryConflictException; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -45,19 +42,11 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - String name = request.param("repository"); - final var deleteRepositoryRequest = new DeleteRepositoryRequest(getMasterNodeTimeout(request), getAckTimeout(request), name); - return channel -> client.admin() - .cluster() - .deleteRepository( - deleteRepositoryRequest, - new RestToXContentListener(channel).delegateResponse((delegate, err) -> { - if (request.getRestApiVersion().equals(RestApiVersion.V_7) && err instanceof RepositoryConflictException) { - delegate.onFailure(new IllegalStateException(((RepositoryConflictException) err).getBackwardCompatibleMessage())); - } else { - delegate.onFailure(err); - } - }) - ); + final var deleteRepositoryRequest = new DeleteRepositoryRequest( + getMasterNodeTimeout(request), + getAckTimeout(request), + request.param("repository") + ); + return channel -> client.admin().cluster().deleteRepository(deleteRepositoryRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java index e84ed3d59be1d..1302247a813f7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -26,7 +25,6 @@ import java.io.IOException; import java.util.List; -import java.util.Locale; import static org.elasticsearch.rest.ChunkedRestResponseBodyPart.fromTextChunks; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -36,64 +34,9 @@ @ServerlessScope(Scope.INTERNAL) public class RestNodesHotThreadsAction extends BaseRestHandler { - private static final String formatDeprecatedMessageWithoutNodeID = "[%s] is a deprecated endpoint. " - + "Please use [/_nodes/hot_threads] instead."; - private static final String formatDeprecatedMessageWithNodeID = "[%s] is a deprecated endpoint. " - + "Please use [/_nodes/{nodeId}/hot_threads] instead."; - private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS = String.format( - Locale.ROOT, - formatDeprecatedMessageWithoutNodeID, - "/_cluster/nodes/hot_threads" - ); - private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS = String.format( - Locale.ROOT, - formatDeprecatedMessageWithNodeID, - "/_cluster/nodes/{nodeId}/hot_threads" - ); - private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS = String.format( - Locale.ROOT, - formatDeprecatedMessageWithoutNodeID, - "/_cluster/nodes/hotthreads" - ); - private static final String DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS = String.format( - Locale.ROOT, - formatDeprecatedMessageWithNodeID, - "/_cluster/nodes/{nodeId}/hotthreads" - ); - private static final String DEPRECATED_MESSAGE_NODES_HOTTHREADS = String.format( - Locale.ROOT, - formatDeprecatedMessageWithoutNodeID, - "/_nodes/hotthreads" - ); - private static final String DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS = String.format( - Locale.ROOT, - formatDeprecatedMessageWithNodeID, - "/_nodes/{nodeId}/hotthreads" - ); - @Override public List routes() { - return List.of( - new Route(GET, "/_nodes/hot_threads"), - new Route(GET, "/_nodes/{nodeId}/hot_threads"), - - Route.builder(GET, "/_cluster/nodes/hot_threads") - .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_HOT_THREADS, RestApiVersion.V_7) - .build(), - Route.builder(GET, "/_cluster/nodes/{nodeId}/hot_threads") - .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOT_THREADS, RestApiVersion.V_7) - .build(), - Route.builder(GET, "/_cluster/nodes/hotthreads") - .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_HOTTHREADS, RestApiVersion.V_7) - .build(), - Route.builder(GET, "/_cluster/nodes/{nodeId}/hotthreads") - .deprecated(DEPRECATED_MESSAGE_CLUSTER_NODES_NODEID_HOTTHREADS, RestApiVersion.V_7) - .build(), - Route.builder(GET, "/_nodes/hotthreads").deprecated(DEPRECATED_MESSAGE_NODES_HOTTHREADS, RestApiVersion.V_7).build(), - Route.builder(GET, "/_nodes/{nodeId}/hotthreads") - .deprecated(DEPRECATED_MESSAGE_NODES_NODEID_HOTTHREADS, RestApiVersion.V_7) - .build() - ); + return List.of(new Route(GET, "/_nodes/hot_threads"), new Route(GET, "/_nodes/{nodeId}/hot_threads")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java index 6f913932f4335..7419a589890e8 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java @@ -10,10 +10,7 @@ package org.elasticsearch.rest.action.admin.cluster; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.repositories.RepositoryConflictException; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -47,23 +44,15 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - String name = request.param("repository"); - final var putRepositoryRequest = new PutRepositoryRequest(getMasterNodeTimeout(request), getAckTimeout(request), name); + final var putRepositoryRequest = new PutRepositoryRequest( + getMasterNodeTimeout(request), + getAckTimeout(request), + request.param("repository") + ); try (XContentParser parser = request.contentParser()) { putRepositoryRequest.source(parser.mapOrdered()); } putRepositoryRequest.verify(request.paramAsBoolean("verify", true)); - return channel -> client.admin() - .cluster() - .putRepository( - putRepositoryRequest, - new RestToXContentListener(channel).delegateResponse((delegate, err) -> { - if (request.getRestApiVersion().equals(RestApiVersion.V_7) && err instanceof RepositoryConflictException) { - delegate.onFailure(new IllegalStateException(((RepositoryConflictException) err).getBackwardCompatibleMessage())); - } else { - delegate.onFailure(err); - } - }) - ); + return channel -> client.admin().cluster().putRepository(putRepositoryRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 66465676ba53c..37391028dbd6e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -45,13 +45,7 @@ public class RestGetFieldMappingAction extends BaseRestHandler { @Override public List routes() { - return List.of( - new Route(GET, "/_mapping/field/{fields}"), - new Route(GET, "/{index}/_mapping/field/{fields}"), - Route.builder(GET, "/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/{type}/_mapping/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(GET, "/_mapping/field/{fields}"), new Route(GET, "/{index}/_mapping/field/{fields}")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 2aa230a13c2a9..5f40bea92f818 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -46,13 +46,8 @@ public List routes() { return List.of( new Route(GET, "/_mapping"), new Route(GET, "/_mappings"), - Route.builder(GET, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), new Route(GET, "/{index}/_mapping"), - new Route(GET, "/{index}/_mappings"), - Route.builder(GET, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(HEAD, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(GET, "/{index}/_mappings") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index ae5e46866f5a6..014e761acc388 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -45,19 +45,7 @@ public List routes() { new Route(POST, "/{index}/_mapping/"), new Route(PUT, "/{index}/_mapping/"), new Route(POST, "/{index}/_mappings/"), - new Route(PUT, "/{index}/_mappings/"), - Route.builder(POST, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_mappings").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/{index}/{type}/_mappings").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(PUT, "/{index}/_mappings/") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java deleted file mode 100644 index d0a4160647f1b..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestChannel; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.action.RestBuilderListener; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.rest.RestRequest.Method.POST; - -public class RestSyncedFlushAction extends BaseRestHandler { - - private static final String DEPRECATION_MESSAGE = - "Synced flush is deprecated and will be removed in 8.0. Use flush at /_flush or /{index}/_flush instead."; - - @Override - public List routes() { - return List.of( - Route.builder(GET, "/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/_flush/synced").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); - } - - @Override - public String getName() { - return "synced_flush_action"; - } - - @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final FlushRequest flushRequest = new FlushRequest(Strings.splitStringByCommaToArray(request.param("index"))); - flushRequest.indicesOptions(IndicesOptions.fromRequest(request, flushRequest.indicesOptions())); - return channel -> client.admin().indices().flush(flushRequest, new SimulateSyncedFlushResponseListener(channel)); - } - - static final class SimulateSyncedFlushResponseListener extends RestBuilderListener { - - SimulateSyncedFlushResponseListener(RestChannel channel) { - super(channel); - } - - @Override - public RestResponse buildResponse(BroadcastResponse flushResponse, XContentBuilder builder) throws Exception { - builder.startObject(); - buildSyncedFlushResponse(builder, flushResponse); - builder.endObject(); - final RestStatus restStatus = flushResponse.getFailedShards() == 0 ? RestStatus.OK : RestStatus.CONFLICT; - return new RestResponse(restStatus, builder); - } - - private static void buildSyncedFlushResponse(XContentBuilder builder, BroadcastResponse flushResponse) throws IOException { - builder.startObject("_shards"); - builder.field("total", flushResponse.getTotalShards()); - builder.field("successful", flushResponse.getSuccessfulShards()); - builder.field("failed", flushResponse.getFailedShards()); - // can't serialize the detail of each index as we don't have the shard count per index. - builder.endObject(); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeActionDeprecated.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeActionDeprecated.java deleted file mode 100644 index 148924f522e5e..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeActionDeprecated.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; - -import java.io.IOException; -import java.util.List; -import java.util.Locale; - -import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.rest.RestRequest.Method.POST; - -public class RestUpgradeActionDeprecated extends BaseRestHandler { - public static final String UPGRADE_API_DEPRECATION_MESSAGE = - "The _upgrade API is no longer useful and will be removed. Instead, see _reindex API."; - - @Override - public List routes() { - return List.of( - Route.builder(POST, "/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/_upgrade").deprecated(UPGRADE_API_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); - } - - @Override - public String getName() { - return "upgrade_action"; - } - - @Override - public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - request.param("index"); - final UpgradeActionDeprecatedException exception = new UpgradeActionDeprecatedException(request); - return channel -> channel.sendResponse(new RestResponse(channel, exception)); - } - - public static class UpgradeActionDeprecatedException extends IllegalArgumentException { - private final String path; - private final RestRequest.Method method; - - public UpgradeActionDeprecatedException(RestRequest restRequest) { - this.path = restRequest.path(); - this.method = restRequest.method(); - } - - @Override - public final String getMessage() { - return String.format(Locale.ROOT, "Upgrade action %s %s was removed, use _reindex API instead", method, path); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java index e5b88d101f11a..8784fad3405d0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java @@ -46,9 +46,7 @@ public List routes() { new Route(GET, "/_validate/query"), new Route(POST, "/_validate/query"), new Route(GET, "/{index}/_validate/query"), - new Route(POST, "/{index}/_validate/query"), - Route.builder(GET, "/{index}/{type}/_validate/query").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_validate/query").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_validate/query") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index 33ee87c0c0b5a..0b8e64f5eab4a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -79,9 +79,7 @@ public List routes() { new Route(POST, "/_bulk"), new Route(PUT, "/_bulk"), new Route(POST, "/{index}/_bulk"), - new Route(PUT, "/{index}/_bulk"), - Route.builder(POST, "/{index}/{type}/_bulk").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/{index}/{type}/_bulk").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(PUT, "/{index}/_bulk") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java index f70335a7d39be..3ee1810967153 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java @@ -29,15 +29,10 @@ @ServerlessScope(Scope.PUBLIC) public class RestDeleteAction extends BaseRestHandler { - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " - + "document index requests is deprecated, use the /{index}/_doc/{id} endpoint instead."; @Override public List routes() { - return List.of( - new Route(DELETE, "/{index}/_doc/{id}"), - Route.builder(DELETE, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(DELETE, "/{index}/_doc/{id}")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java index 9018e3e0806d4..cc2b820eb05f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java @@ -32,9 +32,6 @@ @ServerlessScope(Scope.PUBLIC) public class RestGetAction extends BaseRestHandler { - static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " - + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead."; - @Override public String getName() { return "document_get_action"; @@ -42,12 +39,7 @@ public String getName() { @Override public List routes() { - return List.of( - new Route(GET, "/{index}/_doc/{id}"), - new Route(HEAD, "/{index}/_doc/{id}"), - Route.builder(GET, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(HEAD, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(GET, "/{index}/_doc/{id}"), new Route(HEAD, "/{index}/_doc/{id}")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index a71bf9c305185..e6567d7fdf592 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -47,12 +47,7 @@ public class RestGetSourceAction extends BaseRestHandler { @Override public List routes() { - return List.of( - new Route(GET, "/{index}/_source/{id}"), - new Route(HEAD, "/{index}/_source/{id}"), - Route.builder(GET, "/{index}/{type}/{id}/_source").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(HEAD, "/{index}/{type}/{id}/_source").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(GET, "/{index}/_source/{id}"), new Route(HEAD, "/{index}/_source/{id}")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 2eb0e5a1ef038..9931c10b38f3f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -44,12 +44,7 @@ public class RestIndexAction extends BaseRestHandler { @Override public List routes() { - return List.of( - new Route(POST, "/{index}/_doc/{id}"), - new Route(PUT, "/{index}/_doc/{id}"), - Route.builder(POST, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/{index}/{type}/{id}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(POST, "/{index}/_doc/{id}"), new Route(PUT, "/{index}/_doc/{id}")); } @Override @@ -67,12 +62,7 @@ public String getName() { @Override public List routes() { - return List.of( - new Route(POST, "/{index}/_create/{id}"), - new Route(PUT, "/{index}/_create/{id}"), - Route.builder(POST, "/{index}/{type}/{id}/_create").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(PUT, "/{index}/{type}/{id}/_create").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(POST, "/{index}/_create/{id}"), new Route(PUT, "/{index}/_create/{id}")); } @Override @@ -101,10 +91,7 @@ public String getName() { @Override public List routes() { - return List.of( - new Route(POST, "/{index}/_doc"), - Route.builder(POST, "/{index}/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(POST, "/{index}/_doc")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java index 36c9653f53316..0ceb9ebab7397 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java @@ -44,9 +44,7 @@ public List routes() { new Route(GET, "/_mget"), new Route(POST, "/_mget"), new Route(GET, "/{index}/_mget"), - new Route(POST, "/{index}/_mget"), - Route.builder(GET, "/{index}/{type}/_mget").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_mget").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_mget") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java index 2832b31570723..0e23362e3f5df 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java @@ -39,9 +39,7 @@ public List routes() { new Route(GET, "/_mtermvectors"), new Route(POST, "/_mtermvectors"), new Route(GET, "/{index}/_mtermvectors"), - new Route(POST, "/{index}/_mtermvectors"), - Route.builder(GET, "/{index}/{type}/_mtermvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_mtermvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_mtermvectors") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java index b2bface51ce89..1fbf35856589b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java @@ -44,11 +44,7 @@ public List routes() { new Route(GET, "/{index}/_termvectors"), new Route(POST, "/{index}/_termvectors"), new Route(GET, "/{index}/_termvectors/{id}"), - new Route(POST, "/{index}/_termvectors/{id}"), - Route.builder(GET, "/{index}/{type}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/{type}/{id}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/{id}/_termvectors").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_termvectors/{id}") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java index 7e9b766fb499b..682d5b5c55c3f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java @@ -33,15 +33,9 @@ @ServerlessScope(Scope.PUBLIC) public class RestUpdateAction extends BaseRestHandler { - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " - + "document update requests is deprecated, use the endpoint /{index}/_update/{id} instead."; - @Override public List routes() { - return List.of( - new Route(POST, "/{index}/_update/{id}"), - Route.builder(POST, "/{index}/{type}/{id}/_update").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(POST, "/{index}/_update/{id}")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java index 5556052e79ab8..0c3680e09e6bf 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java @@ -46,9 +46,7 @@ public List routes() { new Route(GET, "/_count"), new Route(POST, "/_count"), new Route(GET, "/{index}/_count"), - new Route(POST, "/{index}/_count"), - Route.builder(GET, "/{index}/{type}/_count").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_count").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_count") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java index 11635bee3008a..4d88d115e1da7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java @@ -34,16 +34,10 @@ */ @ServerlessScope(value = Scope.PUBLIC) public class RestExplainAction extends BaseRestHandler { - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying a type in explain requests is deprecated."; @Override public List routes() { - return List.of( - new Route(GET, "/{index}/_explain/{id}"), - new Route(POST, "/{index}/_explain/{id}"), - Route.builder(GET, "/{index}/{type}/{id}/_explain").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/{id}/_explain").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() - ); + return List.of(new Route(GET, "/{index}/_explain/{id}"), new Route(POST, "/{index}/_explain/{id}")); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index bdd7c6734172d..a58904c2649d9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -65,9 +65,7 @@ public List routes() { new Route(GET, "/_msearch"), new Route(POST, "/_msearch"), new Route(GET, "/{index}/_msearch"), - new Route(POST, "/{index}/_msearch"), - Route.builder(GET, "/{index}/{type}/_msearch").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_msearch").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_msearch") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 38157efd8a370..4fff9229372ea 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -89,9 +89,7 @@ public List routes() { new Route(GET, "/_search"), new Route(POST, "/_search"), new Route(GET, "/{index}/_search"), - new Route(POST, "/{index}/_search"), - Route.builder(GET, "/{index}/{type}/_search").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_search").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + new Route(POST, "/{index}/_search") ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 45fd6afe4fca6..7828bb956a160 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -20,6 +20,11 @@ private SearchCapabilities() {} /** Support regex and range match rules in interval queries. */ private static final String RANGE_REGEX_INTERVAL_QUERY_CAPABILITY = "range_regexp_interval_queries"; + /** Support synthetic source with `bit` type in `dense_vector` field when `index` is set to `false`. */ + private static final String BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY = "bit_dense_vector_synthetic_source"; - public static final Set CAPABILITIES = Set.of(RANGE_REGEX_INTERVAL_QUERY_CAPABILITY); + public static final Set CAPABILITIES = Set.of( + RANGE_REGEX_INTERVAL_QUERY_CAPABILITY, + BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY + ); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java index 9dde32cc75e6a..b27019765e33b 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java @@ -12,9 +12,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermStates; -import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.util.CachedSupplier; import org.elasticsearch.features.NodeFeature; @@ -71,17 +70,15 @@ public int uniqueTermsCount() { public int matchedTermsCount() { final int docId = docIdSupplier.getAsInt(); int matchedTerms = 0; + advancePostings(docId); - try { - for (PostingsEnum postingsEnum : postingsSupplier.get()) { - if (postingsEnum != null && postingsEnum.advance(docId) == docId && postingsEnum.freq() > 0) { - matchedTerms++; - } + for (PostingsEnum postingsEnum : postingsSupplier.get()) { + if (postingsEnum != null && postingsEnum.docID() == docId) { + matchedTerms++; } - return matchedTerms; - } catch (IOException e) { - throw new UncheckedIOException(e); } + + return matchedTerms; } /** @@ -150,8 +147,9 @@ public StatsSummary termFreq() { final int docId = docIdSupplier.getAsInt(); try { + advancePostings(docId); for (PostingsEnum postingsEnum : postingsSupplier.get()) { - if (postingsEnum == null || postingsEnum.advance(docId) != docId) { + if (postingsEnum == null || postingsEnum.docID() != docId) { statsSummary.accept(0); } else { statsSummary.accept(postingsEnum.freq()); @@ -170,12 +168,13 @@ public StatsSummary termFreq() { * @return statistics on termPositions for the terms of the query in the current dac */ public StatsSummary termPositions() { - try { - statsSummary.reset(); - int docId = docIdSupplier.getAsInt(); + statsSummary.reset(); + int docId = docIdSupplier.getAsInt(); + try { + advancePostings(docId); for (PostingsEnum postingsEnum : postingsSupplier.get()) { - if (postingsEnum == null || postingsEnum.advance(docId) != docId) { + if (postingsEnum == null || postingsEnum.docID() != docId) { continue; } for (int i = 0; i < postingsEnum.freq(); i++) { @@ -206,25 +205,9 @@ private TermStates[] loadTermContexts() { private PostingsEnum[] loadPostings() { try { PostingsEnum[] postings = new PostingsEnum[terms.length]; - TermStates[] contexts = termContextsSupplier.get(); for (int i = 0; i < terms.length; i++) { - TermStates termStates = contexts[i]; - if (termStates.docFreq() == 0) { - postings[i] = null; - continue; - } - - TermState state = termStates.get(leafReaderContext); - if (state == null) { - postings[i] = null; - continue; - } - - TermsEnum termsEnum = leafReaderContext.reader().terms(terms[i].field()).iterator(); - termsEnum.seekExact(terms[i].bytes(), state); - - postings[i] = termsEnum.postings(null, PostingsEnum.ALL); + postings[i] = leafReaderContext.reader().postings(terms[i], PostingsEnum.POSITIONS); } return postings; @@ -232,4 +215,16 @@ private PostingsEnum[] loadPostings() { throw new UncheckedIOException(e); } } + + private void advancePostings(int targetDocId) { + try { + for (PostingsEnum posting : postingsSupplier.get()) { + if (posting != null && posting.docID() < targetDocId && posting.docID() != DocIdSetIterator.NO_MORE_DOCS) { + posting.advance(targetDocId); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index be96b4e25d841..3a900a8a9b8a6 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedSupplier; @@ -1618,7 +1619,7 @@ public boolean isForceExecution() { } } - public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { + public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { return indicesService.buildAliasFilter(state, index, resolvedExpressions); } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java index fb5015a82dbdb..b78d9e40ba120 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java @@ -107,11 +107,10 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio @Override public Scorer scorer(LeafReaderContext context) { - // Segment starts indicate how many docs are in the segment, - // upper equalling lower indicates no documents for this segment - if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) { - return null; - } + /** + * We return a scorer even if there are no ranked documents within the segment. + * This ensures the correct propagation of the maximum score. + */ return new Scorer(this) { final int lower = segmentStarts[context.ord]; final int upper = segmentStarts[context.ord + 1]; diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java b/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java index 6707d77d6a2d0..d49ac1e29bea6 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java @@ -115,7 +115,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (description != null) { builder.field("description", description); } - builder.timeField("start_time_in_millis", "start_time", startTime); + builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTime); if (builder.humanReadable()) { builder.field("running_time", new TimeValue(runningTimeNanos, TimeUnit.NANOSECONDS).toString()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 67e5f30f023c9..b59cc13a20ff2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -37,7 +37,6 @@ import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -323,21 +322,6 @@ private void assertXContent( AbstractChunkedSerializingTestCase.assertChunkCount(response, params, o -> expectedChunks[0]); assertCriticalWarnings(criticalDeprecationWarnings); - - // check the v7 API too - AbstractChunkedSerializingTestCase.assertChunkCount(new ChunkedToXContent() { - @Override - public Iterator toXContentChunked(ToXContent.Params outerParams) { - return response.toXContentChunkedV7(outerParams); - } - - @Override - public boolean isFragment() { - return response.isFragment(); - } - }, params, o -> expectedChunks[0]++); - // the v7 API should not emit any deprecation warnings - assertCriticalWarnings(); } private static ClusterRerouteResponse createClusterRerouteResponse(ClusterState clusterState) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index 834bacd9e6a04..1faeabb6acbf7 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -229,9 +230,19 @@ public void testResolveHiddenProperlyWithDateMath() { .metadata(buildMetadata(new Object[][] {}, indices)) .build(); String[] requestedIndex = new String[] { "" }; - Set resolvedIndices = resolver.resolveExpressions(clusterState, IndicesOptions.LENIENT_EXPAND_OPEN, true, requestedIndex); + Set resolvedIndices = resolver.resolveExpressions( + clusterState, + IndicesOptions.LENIENT_EXPAND_OPEN, + true, + requestedIndex + ); assertThat(resolvedIndices.size(), is(1)); - assertThat(resolvedIndices, contains(oneOf("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix))); + assertThat( + resolvedIndices, + contains( + oneOf(new ResolvedExpression("logs-pgsql-prod-" + todaySuffix), new ResolvedExpression("logs-pgsql-prod-" + tomorrowSuffix)) + ) + ); } public void testSystemIndexAccess() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 8f0ff82beab4b..74408b99e92ce 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; @@ -69,7 +70,7 @@ public void testSettingsProviderIsOverridden() throws Exception { public Settings getAdditionalIndexSettings( String indexName, String dataStreamName, - boolean timeSeries, + IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, Settings allSettings, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java index 020955d226a0f..9b0eb93b496a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java @@ -24,7 +24,16 @@ protected Writeable.Reader instanceReader() { @Override protected DataStreamOptions createTestInstance() { - return new DataStreamOptions(randomBoolean() ? null : DataStreamFailureStoreTests.randomFailureStore()); + return randomDataStreamOptions(); + } + + public static DataStreamOptions randomDataStreamOptions() { + return switch (randomIntBetween(0, 2)) { + case 0 -> DataStreamOptions.EMPTY; + case 1 -> DataStreamOptions.FAILURE_STORE_DISABLED; + case 2 -> DataStreamOptions.FAILURE_STORE_ENABLED; + default -> throw new IllegalArgumentException("Illegal randomisation branch"); + }; } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java index 6be5b48f9d723..fe0b7926229cb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.DateMathExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; @@ -26,7 +27,6 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; @@ -52,11 +52,11 @@ private static String formatDate(String pattern, ZonedDateTime zonedDateTime) { public void testNormal() throws Exception { int numIndexExpressions = randomIntBetween(1, 9); - List indexExpressions = new ArrayList<>(numIndexExpressions); + List indexExpressions = new ArrayList<>(numIndexExpressions); for (int i = 0; i < numIndexExpressions; i++) { - indexExpressions.add(randomAlphaOfLength(10)); + indexExpressions.add(new ResolvedExpression(randomAlphaOfLength(10))); } - List result = DateMathExpressionResolver.resolve(context, indexExpressions); + List result = DateMathExpressionResolver.resolve(context, indexExpressions); assertThat(result.size(), equalTo(indexExpressions.size())); for (int i = 0; i < indexExpressions.size(); i++) { assertThat(result.get(i), equalTo(indexExpressions.get(i))); @@ -64,25 +64,25 @@ public void testNormal() throws Exception { } public void testExpression() throws Exception { - List indexExpressions = Arrays.asList("<.marvel-{now}>", "<.watch_history-{now}>", ""); - List result = DateMathExpressionResolver.resolve(context, indexExpressions); + List indexExpressions = resolvedExpressions("<.marvel-{now}>", "<.watch_history-{now}>", ""); + List result = DateMathExpressionResolver.resolve(context, indexExpressions); assertThat(result.size(), equalTo(3)); - assertThat(result.get(0), equalTo(".marvel-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); - assertThat(result.get(1), equalTo(".watch_history-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); - assertThat(result.get(2), equalTo("logstash-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); + assertThat(result.get(0).resource(), equalTo(".marvel-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); + assertThat(result.get(1).resource(), equalTo(".watch_history-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); + assertThat(result.get(2).resource(), equalTo("logstash-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); } public void testExpressionWithWildcardAndExclusions() { - List indexExpressions = Arrays.asList( + List indexExpressions = resolvedExpressions( "<-before-inner-{now}>", "-", "", "<-after-inner-{now}>", "-" ); - List result = DateMathExpressionResolver.resolve(context, indexExpressions); + List result = DateMathExpressionResolver.resolve(context, indexExpressions); assertThat( - result, + result.stream().map(ResolvedExpression::resource).toList(), Matchers.contains( equalTo("-before-inner-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime()))), equalTo("-"), // doesn't evaluate because it doesn't start with "<" and it is not an exclusion @@ -98,7 +98,7 @@ public void testExpressionWithWildcardAndExclusions() { ); result = DateMathExpressionResolver.resolve(noWildcardExpandContext, indexExpressions); assertThat( - result, + result.stream().map(ResolvedExpression::resource).toList(), Matchers.contains( equalTo("-before-inner-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime()))), // doesn't evaluate because it doesn't start with "<" and there can't be exclusions without wildcard expansion @@ -112,21 +112,24 @@ public void testExpressionWithWildcardAndExclusions() { } public void testEmpty() throws Exception { - List result = DateMathExpressionResolver.resolve(context, Collections.emptyList()); + List result = DateMathExpressionResolver.resolve(context, List.of()); assertThat(result.size(), equalTo(0)); } public void testExpression_Static() throws Exception { - List result = DateMathExpressionResolver.resolve(context, Arrays.asList("<.marvel-test>")); + List result = DateMathExpressionResolver.resolve(context, resolvedExpressions("<.marvel-test>")); assertThat(result.size(), equalTo(1)); - assertThat(result.get(0), equalTo(".marvel-test")); + assertThat(result.get(0).resource(), equalTo(".marvel-test")); } public void testExpression_MultiParts() throws Exception { - List result = DateMathExpressionResolver.resolve(context, Arrays.asList("<.text1-{now/d}-text2-{now/M}>")); + List result = DateMathExpressionResolver.resolve( + context, + resolvedExpressions("<.text1-{now/d}-text2-{now/M}>") + ); assertThat(result.size(), equalTo(1)); assertThat( - result.get(0), + result.get(0).resource(), equalTo( ".text1-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())) @@ -137,33 +140,42 @@ public void testExpression_MultiParts() throws Exception { } public void testExpression_CustomFormat() throws Exception { - List results = DateMathExpressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{yyyy.MM.dd}}>")); + List results = DateMathExpressionResolver.resolve( + context, + resolvedExpressions("<.marvel-{now/d{yyyy.MM.dd}}>") + ); assertThat(results.size(), equalTo(1)); - assertThat(results.get(0), equalTo(".marvel-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); + assertThat(results.get(0).resource(), equalTo(".marvel-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); } public void testExpression_EscapeStatic() throws Exception { - List result = DateMathExpressionResolver.resolve(context, Arrays.asList("<.mar\\{v\\}el-{now/d}>")); + List result = DateMathExpressionResolver.resolve(context, resolvedExpressions("<.mar\\{v\\}el-{now/d}>")); assertThat(result.size(), equalTo(1)); - assertThat(result.get(0), equalTo(".mar{v}el-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); + assertThat(result.get(0).resource(), equalTo(".mar{v}el-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); } public void testExpression_EscapeDateFormat() throws Exception { - List result = DateMathExpressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{'\\{year\\}'yyyy}}>")); + List result = DateMathExpressionResolver.resolve( + context, + resolvedExpressions("<.marvel-{now/d{'\\{year\\}'yyyy}}>") + ); assertThat(result.size(), equalTo(1)); - assertThat(result.get(0), equalTo(".marvel-" + formatDate("'{year}'yyyy", dateFromMillis(context.getStartTime())))); + assertThat(result.get(0).resource(), equalTo(".marvel-" + formatDate("'{year}'yyyy", dateFromMillis(context.getStartTime())))); } public void testExpression_MixedArray() throws Exception { - List result = DateMathExpressionResolver.resolve( + List result = DateMathExpressionResolver.resolve( context, - Arrays.asList("name1", "<.marvel-{now/d}>", "name2", "<.logstash-{now/M{uuuu.MM}}>") + resolvedExpressions("name1", "<.marvel-{now/d}>", "name2", "<.logstash-{now/M{uuuu.MM}}>") ); assertThat(result.size(), equalTo(4)); - assertThat(result.get(0), equalTo("name1")); - assertThat(result.get(1), equalTo(".marvel-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); - assertThat(result.get(2), equalTo("name2")); - assertThat(result.get(3), equalTo(".logstash-" + formatDate("uuuu.MM", dateFromMillis(context.getStartTime()).withDayOfMonth(1)))); + assertThat(result.get(0).resource(), equalTo("name1")); + assertThat(result.get(1).resource(), equalTo(".marvel-" + formatDate("uuuu.MM.dd", dateFromMillis(context.getStartTime())))); + assertThat(result.get(2).resource(), equalTo("name2")); + assertThat( + result.get(3).resource(), + equalTo(".logstash-" + formatDate("uuuu.MM", dateFromMillis(context.getStartTime()).withDayOfMonth(1))) + ); } public void testExpression_CustomTimeZoneInIndexName() throws Exception { @@ -202,19 +214,19 @@ public void testExpression_CustomTimeZoneInIndexName() throws Exception { name -> false, name -> false ); - List results = DateMathExpressionResolver.resolve( + List results = DateMathExpressionResolver.resolve( context, - Arrays.asList("<.marvel-{now/d{yyyy.MM.dd|" + timeZone.getId() + "}}>") + resolvedExpressions("<.marvel-{now/d{yyyy.MM.dd|" + timeZone.getId() + "}}>") ); assertThat(results.size(), equalTo(1)); logger.info("timezone: [{}], now [{}], name: [{}]", timeZone, now, results.get(0)); - assertThat(results.get(0), equalTo(".marvel-" + formatDate("uuuu.MM.dd", now.withZoneSameInstant(timeZone)))); + assertThat(results.get(0).resource(), equalTo(".marvel-" + formatDate("uuuu.MM.dd", now.withZoneSameInstant(timeZone)))); } public void testExpressionInvalidUnescaped() throws Exception { Exception e = expectThrows( ElasticsearchParseException.class, - () -> DateMathExpressionResolver.resolve(context, Arrays.asList("<.mar}vel-{now/d}>")) + () -> DateMathExpressionResolver.resolve(context, resolvedExpressions("<.mar}vel-{now/d}>")) ); assertThat(e.getMessage(), containsString("invalid dynamic name expression")); assertThat(e.getMessage(), containsString("invalid character at position [")); @@ -223,7 +235,7 @@ public void testExpressionInvalidUnescaped() throws Exception { public void testExpressionInvalidDateMathFormat() throws Exception { Exception e = expectThrows( ElasticsearchParseException.class, - () -> DateMathExpressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}>")) + () -> DateMathExpressionResolver.resolve(context, resolvedExpressions("<.marvel-{now/d{}>")) ); assertThat(e.getMessage(), containsString("invalid dynamic name expression")); assertThat(e.getMessage(), containsString("date math placeholder is open ended")); @@ -232,7 +244,7 @@ public void testExpressionInvalidDateMathFormat() throws Exception { public void testExpressionInvalidEmptyDateMathFormat() throws Exception { Exception e = expectThrows( ElasticsearchParseException.class, - () -> DateMathExpressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}}>")) + () -> DateMathExpressionResolver.resolve(context, resolvedExpressions("<.marvel-{now/d{}}>")) ); assertThat(e.getMessage(), containsString("invalid dynamic name expression")); assertThat(e.getMessage(), containsString("missing date format")); @@ -241,10 +253,13 @@ public void testExpressionInvalidEmptyDateMathFormat() throws Exception { public void testExpressionInvalidOpenEnded() throws Exception { Exception e = expectThrows( ElasticsearchParseException.class, - () -> DateMathExpressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d>")) + () -> DateMathExpressionResolver.resolve(context, resolvedExpressions("<.marvel-{now/d>")) ); assertThat(e.getMessage(), containsString("invalid dynamic name expression")); assertThat(e.getMessage(), containsString("date math placeholder is open ended")); } + private List resolvedExpressions(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).toList(); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java index 1ca59ff402bd8..1df3bf4132b60 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java @@ -13,10 +13,12 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ExpressionList; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ExpressionList.Expression; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.function.Supplier; @@ -39,10 +41,13 @@ public void testEmpty() { public void testExplicitSingleNameExpression() { for (IndicesOptions indicesOptions : List.of(getExpandWildcardsIndicesOptions(), getNoExpandWildcardsIndicesOptions())) { for (String expressionString : List.of("non_wildcard", "-non_exclusion")) { - ExpressionList expressionList = new ExpressionList(getContextWithOptions(indicesOptions), List.of(expressionString)); + ExpressionList expressionList = new ExpressionList( + getContextWithOptions(indicesOptions), + resolvedExpressions(expressionString) + ); assertThat(expressionList.hasWildcard(), is(false)); if (randomBoolean()) { - expressionList = new ExpressionList(getContextWithOptions(indicesOptions), List.of(expressionString)); + expressionList = new ExpressionList(getContextWithOptions(indicesOptions), resolvedExpressions((expressionString))); } Iterator expressionIterator = expressionList.iterator(); assertThat(expressionIterator.hasNext(), is(true)); @@ -62,11 +67,14 @@ public void testWildcardSingleExpression() { for (String wildcardTest : List.of("*", "a*", "*b", "a*b", "a-*b", "a*-b", "-*", "-a*", "-*b", "**", "*-*")) { ExpressionList expressionList = new ExpressionList( getContextWithOptions(getExpandWildcardsIndicesOptions()), - List.of(wildcardTest) + resolvedExpressions(wildcardTest) ); assertThat(expressionList.hasWildcard(), is(true)); if (randomBoolean()) { - expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), List.of(wildcardTest)); + expressionList = new ExpressionList( + getContextWithOptions(getExpandWildcardsIndicesOptions()), + resolvedExpressions(wildcardTest) + ); } Iterator expressionIterator = expressionList.iterator(); assertThat(expressionIterator.hasNext(), is(true)); @@ -82,13 +90,13 @@ public void testWildcardSingleExpression() { } public void testWildcardLongerExpression() { - List onlyExplicits = randomList(7, () -> randomAlphaOfLengthBetween(0, 5)); - String wildcard = randomFrom("*", "*b", "-*", "*-", "c*", "a*b", "**"); - List expressionList = new ArrayList<>(onlyExplicits.size() + 1); + List onlyExplicits = randomList(7, () -> new ResolvedExpression(randomAlphaOfLengthBetween(0, 5))); + ResolvedExpression wildcard = new ResolvedExpression(randomFrom("*", "*b", "-*", "*-", "c*", "a*b", "**")); + List expressionList = new ArrayList<>(onlyExplicits.size() + 1); expressionList.addAll(randomSubsetOf(onlyExplicits)); int wildcardPos = expressionList.size(); expressionList.add(wildcard); - for (String item : onlyExplicits) { + for (ResolvedExpression item : onlyExplicits) { if (expressionList.contains(item) == false) { expressionList.add(item); } @@ -106,18 +114,18 @@ public void testWildcardLongerExpression() { } else { assertThat(expression.isWildcard(), is(true)); } - assertThat(expression.get(), is(expressionList.get(i++))); + assertThat(expression.get(), is(expressionList.get(i++).resource())); } } public void testWildcardsNoExclusionExpressions() { - for (List wildcardExpression : List.of( - List.of("*"), - List.of("a", "*"), - List.of("-b", "*c"), - List.of("-", "a", "c*"), - List.of("*", "a*", "*b"), - List.of("-*", "a", "b*") + for (List wildcardExpression : List.of( + resolvedExpressions("*"), + resolvedExpressions("a", "*"), + resolvedExpressions("-b", "*c"), + resolvedExpressions("-", "a", "c*"), + resolvedExpressions("*", "a*", "*b"), + resolvedExpressions("-*", "a", "b*") )) { ExpressionList expressionList = new ExpressionList( getContextWithOptions(getExpandWildcardsIndicesOptions()), @@ -130,25 +138,25 @@ public void testWildcardsNoExclusionExpressions() { int i = 0; for (Expression expression : expressionList) { assertThat(expression.isExclusion(), is(false)); - if (wildcardExpression.get(i).contains("*")) { + if (wildcardExpression.get(i).resource().contains("*")) { assertThat(expression.isWildcard(), is(true)); } else { assertThat(expression.isWildcard(), is(false)); } - assertThat(expression.get(), is(wildcardExpression.get(i++))); + assertThat(expression.get(), is(wildcardExpression.get(i++).resource())); } } } public void testWildcardExpressionNoExpandOptions() { - for (List wildcardExpression : List.of( - List.of("*"), - List.of("a", "*"), - List.of("-b", "*c"), - List.of("*d", "-"), - List.of("*", "-*"), - List.of("-", "a", "c*"), - List.of("*", "a*", "*b") + for (List wildcardExpression : List.of( + resolvedExpressions("*"), + resolvedExpressions("a", "*"), + resolvedExpressions("-b", "*c"), + resolvedExpressions("*d", "-"), + resolvedExpressions("*", "-*"), + resolvedExpressions("-", "a", "c*"), + resolvedExpressions("*", "a*", "*b") )) { ExpressionList expressionList = new ExpressionList( getContextWithOptions(getNoExpandWildcardsIndicesOptions()), @@ -162,7 +170,7 @@ public void testWildcardExpressionNoExpandOptions() { for (Expression expression : expressionList) { assertThat(expression.isWildcard(), is(false)); assertThat(expression.isExclusion(), is(false)); - assertThat(expression.get(), is(wildcardExpression.get(i++))); + assertThat(expression.get(), is(wildcardExpression.get(i++).resource())); } } } @@ -172,17 +180,17 @@ public void testSingleExclusionExpression() { int wildcardPos = randomIntBetween(0, 3); String exclusion = randomFrom("-*", "-", "-c*", "-ab", "--"); int exclusionPos = randomIntBetween(wildcardPos + 1, 7); - List exclusionExpression = new ArrayList<>(); + List exclusionExpression = new ArrayList<>(); for (int i = 0; i < wildcardPos; i++) { - exclusionExpression.add(randomAlphaOfLengthBetween(0, 5)); + exclusionExpression.add(new ResolvedExpression(randomAlphaOfLengthBetween(0, 5))); } - exclusionExpression.add(wildcard); + exclusionExpression.add(new ResolvedExpression(wildcard)); for (int i = wildcardPos + 1; i < exclusionPos; i++) { - exclusionExpression.add(randomAlphaOfLengthBetween(0, 5)); + exclusionExpression.add(new ResolvedExpression(randomAlphaOfLengthBetween(0, 5))); } - exclusionExpression.add(exclusion); + exclusionExpression.add(new ResolvedExpression(exclusion)); for (int i = 0; i < randomIntBetween(0, 3); i++) { - exclusionExpression.add(randomAlphaOfLengthBetween(0, 5)); + exclusionExpression.add(new ResolvedExpression(randomAlphaOfLengthBetween(0, 5))); } ExpressionList expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), exclusionExpression); if (randomBoolean()) { @@ -193,28 +201,28 @@ public void testSingleExclusionExpression() { if (i == wildcardPos) { assertThat(expression.isWildcard(), is(true)); assertThat(expression.isExclusion(), is(false)); - assertThat(expression.get(), is(exclusionExpression.get(i++))); + assertThat(expression.get(), is(exclusionExpression.get(i++).resource())); } else if (i == exclusionPos) { assertThat(expression.isExclusion(), is(true)); - assertThat(expression.isWildcard(), is(exclusionExpression.get(i).contains("*"))); - assertThat(expression.get(), is(exclusionExpression.get(i++).substring(1))); + assertThat(expression.isWildcard(), is(exclusionExpression.get(i).resource().contains("*"))); + assertThat(expression.get(), is(exclusionExpression.get(i++).resource().substring(1))); } else { assertThat(expression.isWildcard(), is(false)); assertThat(expression.isExclusion(), is(false)); - assertThat(expression.get(), is(exclusionExpression.get(i++))); + assertThat(expression.get(), is(exclusionExpression.get(i++).resource())); } } } public void testExclusionsExpression() { - for (Tuple, List> exclusionExpression : List.of( - new Tuple<>(List.of("-a", "*", "-a"), List.of(false, false, true)), - new Tuple<>(List.of("-b*", "c", "-a"), List.of(false, false, true)), - new Tuple<>(List.of("*d", "-", "*b"), List.of(false, true, false)), - new Tuple<>(List.of("-", "--", "-*", "", "-*"), List.of(false, false, false, false, true)), - new Tuple<>(List.of("*-", "-*", "a", "-b"), List.of(false, true, false, true)), - new Tuple<>(List.of("a", "-b", "-*", "-b", "*", "-b"), List.of(false, false, false, true, false, true)), - new Tuple<>(List.of("-a", "*d", "-a", "-*b", "-b", "--"), List.of(false, false, true, true, true, true)) + for (Tuple, List> exclusionExpression : List.of( + new Tuple<>(resolvedExpressions("-a", "*", "-a"), List.of(false, false, true)), + new Tuple<>(resolvedExpressions("-b*", "c", "-a"), List.of(false, false, true)), + new Tuple<>(resolvedExpressions("*d", "-", "*b"), List.of(false, true, false)), + new Tuple<>(resolvedExpressions("-", "--", "-*", "", "-*"), List.of(false, false, false, false, true)), + new Tuple<>(resolvedExpressions("*-", "-*", "a", "-b"), List.of(false, true, false, true)), + new Tuple<>(resolvedExpressions("a", "-b", "-*", "-b", "*", "-b"), List.of(false, false, false, true, false, true)), + new Tuple<>(resolvedExpressions("-a", "*d", "-a", "-*b", "-b", "--"), List.of(false, false, true, true, true, true)) )) { ExpressionList expressionList = new ExpressionList( getContextWithOptions(getExpandWildcardsIndicesOptions()), @@ -227,11 +235,11 @@ public void testExclusionsExpression() { for (Expression expression : expressionList) { boolean isExclusion = exclusionExpression.v2().get(i); assertThat(expression.isExclusion(), is(isExclusion)); - assertThat(expression.isWildcard(), is(exclusionExpression.v1().get(i).contains("*"))); + assertThat(expression.isWildcard(), is(exclusionExpression.v1().get(i).resource().contains("*"))); if (isExclusion) { - assertThat(expression.get(), is(exclusionExpression.v1().get(i++).substring(1))); + assertThat(expression.get(), is(exclusionExpression.v1().get(i++).resource().substring(1))); } else { - assertThat(expression.get(), is(exclusionExpression.v1().get(i++))); + assertThat(expression.get(), is(exclusionExpression.v1().get(i++).resource())); } } } @@ -306,4 +314,8 @@ private Context getContextWithOptions(IndicesOptions indicesOptions) { when(context.getOptions()).thenReturn(indicesOptions); return context; } + + private List resolvedExpressions(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).toList(); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 5f55d203e00e4..bddbe259e0ef3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata.State; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -1580,16 +1581,27 @@ public void testResolveExpressions() { .put(indexBuilder("test-1").state(State.OPEN).putAlias(AliasMetadata.builder("alias-1"))); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - assertEquals(new HashSet<>(Arrays.asList("alias-0", "alias-1")), indexNameExpressionResolver.resolveExpressions(state, "alias-*")); assertEquals( - new HashSet<>(Arrays.asList("test-0", "alias-0", "alias-1")), + Set.of(new ResolvedExpression("alias-0"), new ResolvedExpression("alias-1")), + indexNameExpressionResolver.resolveExpressions(state, "alias-*") + ); + assertEquals( + Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("alias-0"), new ResolvedExpression("alias-1")), indexNameExpressionResolver.resolveExpressions(state, "test-0", "alias-*") ); assertEquals( - new HashSet<>(Arrays.asList("test-0", "test-1", "alias-0", "alias-1")), + Set.of( + new ResolvedExpression("test-0"), + new ResolvedExpression("test-1"), + new ResolvedExpression("alias-0"), + new ResolvedExpression("alias-1") + ), indexNameExpressionResolver.resolveExpressions(state, "test-*", "alias-*") ); - assertEquals(new HashSet<>(Arrays.asList("test-1", "alias-1")), indexNameExpressionResolver.resolveExpressions(state, "*-1")); + assertEquals( + Set.of(new ResolvedExpression("test-1"), new ResolvedExpression("alias-1")), + indexNameExpressionResolver.resolveExpressions(state, "*-1") + ); } public void testFilteringAliases() { @@ -1598,16 +1610,25 @@ public void testFilteringAliases() { .put(indexBuilder("test-1").state(State.OPEN).putAlias(AliasMetadata.builder("alias-1"))); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - Set resolvedExpressions = new HashSet<>(Arrays.asList("alias-0", "alias-1")); + Set resolvedExpressions = Set.of(new ResolvedExpression("alias-0"), new ResolvedExpression("alias-1")); String[] strings = indexNameExpressionResolver.filteringAliases(state, "test-0", resolvedExpressions); assertArrayEquals(new String[] { "alias-0" }, strings); // concrete index supersedes filtering alias - resolvedExpressions = new HashSet<>(Arrays.asList("test-0", "alias-0", "alias-1")); + resolvedExpressions = Set.of( + new ResolvedExpression("test-0"), + new ResolvedExpression("alias-0"), + new ResolvedExpression("alias-1") + ); strings = indexNameExpressionResolver.filteringAliases(state, "test-0", resolvedExpressions); assertNull(strings); - resolvedExpressions = new HashSet<>(Arrays.asList("test-0", "test-1", "alias-0", "alias-1")); + resolvedExpressions = Set.of( + new ResolvedExpression("test-0"), + new ResolvedExpression("test-1"), + new ResolvedExpression("alias-0"), + new ResolvedExpression("alias-1") + ); strings = indexNameExpressionResolver.filteringAliases(state, "test-0", resolvedExpressions); assertNull(strings); } @@ -1621,7 +1642,7 @@ public void testIndexAliases() { .putAlias(AliasMetadata.builder("test-alias-non-filtering")) ); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "test-*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "test-*"); String[] strings = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, true, resolvedExpressions); Arrays.sort(strings); @@ -1656,28 +1677,28 @@ public void testIndexAliasesDataStreamAliases() { ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { // Only resolve aliases with with that refer to dataStreamName1 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, true, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_foo", "logs", "logs_bar")); } { // Only resolve aliases with with that refer to dataStreamName2 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex2.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, true, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_baz", "logs_baz2")); } { // Null is returned, because skipping identity check and resolvedExpressions contains the backing index name - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex2.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, false, resolvedExpressions); assertThat(result, nullValue()); } { // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1691,7 +1712,7 @@ public void testIndexAliasesDataStreamAliases() { } { // Null is returned, because an unfiltered alias is targeting the same data stream - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "logs_bar", "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "logs_bar", "logs"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1705,7 +1726,7 @@ public void testIndexAliasesDataStreamAliases() { } { // The filtered alias is returned because although we target the data stream name, skipIdentity is true - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1719,7 +1740,7 @@ public void testIndexAliasesDataStreamAliases() { } { // Null is returned because we target the data stream name and skipIdentity is false - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1742,13 +1763,13 @@ public void testIndexAliasesSkipIdentity() { ); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - Set resolvedExpressions = new HashSet<>(Arrays.asList("test-0", "test-alias")); + Set resolvedExpressions = Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-alias")); String[] aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, false, resolvedExpressions); assertNull(aliases); aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, true, resolvedExpressions); assertArrayEquals(new String[] { "test-alias" }, aliases); - resolvedExpressions = Collections.singleton("other-alias"); + resolvedExpressions = Collections.singleton(new ResolvedExpression("other-alias")); aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, false, resolvedExpressions); assertArrayEquals(new String[] { "other-alias" }, aliases); aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, true, resolvedExpressions); @@ -1769,7 +1790,7 @@ public void testConcreteWriteIndexSuccessful() { x -> true, x -> true, true, - new HashSet<>(Arrays.asList("test-0", "test-alias")) + Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-alias")) ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1851,7 +1872,7 @@ public void testConcreteWriteIndexWithWildcardExpansion() { x -> true, x -> true, true, - new HashSet<>(Arrays.asList("test-0", "test-1", "test-alias")) + Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-1"), new ResolvedExpression("test-alias")) ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1889,7 +1910,7 @@ public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() { x -> true, x -> true, true, - new HashSet<>(Arrays.asList("test-0", "test-alias")) + Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-alias")) ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1925,7 +1946,7 @@ public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { x -> true, x -> true, true, - new HashSet<>(Arrays.asList("test-0", "test-1", "test-alias")) + Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-1"), new ResolvedExpression("test-alias")) ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1966,7 +1987,7 @@ public void testAliasResolutionNotAllowingMultipleIndices() { x -> true, x -> true, true, - new HashSet<>(Arrays.asList("test-0", "test-1", "test-alias")) + Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-1"), new ResolvedExpression("test-alias")) ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java index 92c1103c950c0..276c20d2d1322 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java @@ -422,6 +422,39 @@ public void testUpdateLifecycle() { } } + public void testUpdateDataStreamOptions() { + String dataStream = randomAlphaOfLength(5); + // we want the data stream options to be non-empty, so we can see the removal in action + DataStreamOptions dataStreamOptions = randomValueOtherThan( + DataStreamOptions.EMPTY, + DataStreamOptionsTests::randomDataStreamOptions + ); + ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStream, 2)), List.of()); + MetadataDataStreamsService service = new MetadataDataStreamsService( + mock(ClusterService.class), + mock(IndicesService.class), + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) + ); + + // Ensure no data stream options are stored + DataStream updatedDataStream = before.metadata().dataStreams().get(dataStream); + assertNotNull(updatedDataStream); + assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY)); + + // Set non-empty data stream options + ClusterState after = service.updateDataStreamOptions(before, List.of(dataStream), dataStreamOptions); + updatedDataStream = after.metadata().dataStreams().get(dataStream); + assertNotNull(updatedDataStream); + assertThat(updatedDataStream.getDataStreamOptions(), equalTo(dataStreamOptions)); + before = after; + + // Remove data stream options + after = service.updateDataStreamOptions(before, List.of(dataStream), null); + updatedDataStream = after.metadata().dataStreams().get(dataStream); + assertNotNull(updatedDataStream); + assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY)); + } + private MapperService getMapperService(IndexMetadata im) { try { String mapping = im.mapping().source().toString(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 00e21603ec8b4..ba1f9f01f49d2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -2412,30 +2413,87 @@ public void testEnsureMetadataFieldCheckedForGlobalStateChanges() { assertThat(unclassifiedFields, empty()); } - public void testIsTimeSeriesTemplate() throws IOException { - var template = new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("{}"), null); + public void testRetrieveIndexModeFromTemplateTsdb() throws IOException { + // tsdb: + var tsdbTemplate = new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("{}"), null); // Settings in component template: { - var componentTemplate = new ComponentTemplate(template, null, null); + var componentTemplate = new ComponentTemplate(tsdbTemplate, null, null); var indexTemplate = ComposableIndexTemplate.builder() .indexPatterns(List.of("test-*")) .componentTemplates(List.of("component_template_1")) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build(); Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.isTimeSeriesTemplate(indexTemplate), is(true)); + assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES)); } // Settings in composable index template: { var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); var indexTemplate = ComposableIndexTemplate.builder() .indexPatterns(List.of("test-*")) - .template(template) + .template(tsdbTemplate) .componentTemplates(List.of("component_template_1")) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build(); Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.isTimeSeriesTemplate(indexTemplate), is(true)); + assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES)); + } + } + + public void testRetrieveIndexModeFromTemplateLogsdb() throws IOException { + // logsdb: + var logsdbTemplate = new Template(Settings.builder().put("index.mode", "logsdb").build(), new CompressedXContent("{}"), null); + // Settings in component template: + { + var componentTemplate = new ComponentTemplate(logsdbTemplate, null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); + assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB)); + } + // Settings in composable index template: + { + var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(logsdbTemplate) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); + assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB)); + } + } + + public void testRetrieveIndexModeFromTemplateEmpty() throws IOException { + // no index mode: + var emptyTemplate = new Template(Settings.EMPTY, new CompressedXContent("{}"), null); + // Settings in component template: + { + var componentTemplate = new ComponentTemplate(emptyTemplate, null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); + assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), nullValue()); + } + // Settings in composable index template: + { + var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(emptyTemplate) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); + assertThat(m.retrieveIndexModeFromTemplate(indexTemplate), nullValue()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index 982394ca31b1c..25ed5fb2bdab2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata.State; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; @@ -20,13 +21,13 @@ import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; import static org.elasticsearch.common.util.set.Sets.newHashSet; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -50,50 +51,52 @@ public void testConvertWildcardsJustIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("testXXX"))), - equalTo(newHashSet("testXXX")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testXXX"))), + equalTo(resolvedExpressionsSet("testXXX")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testXXX", "testYYY"))), - equalTo(newHashSet("testXXX", "testYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testXXX", "testYYY"))), + equalTo(resolvedExpressionsSet("testXXX", "testYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testXXX", "ku*"))), - equalTo(newHashSet("testXXX", "kuku")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testXXX", "ku*"))), + equalTo(resolvedExpressionsSet("testXXX", "kuku")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("test*"))), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("test*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("testX*"))), - equalTo(newHashSet("testXXX", "testXYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testX*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testX*", "kuku"))), - equalTo(newHashSet("testXXX", "testXYY", "kuku")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testX*", "kuku"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "kuku")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("*"))), - equalTo(newHashSet("testXXX", "testXYY", "testYYY", "kuku")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY", "kuku")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("*", "-kuku"))), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("*", "-kuku"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( newHashSet( IndexNameExpressionResolver.WildcardExpressionResolver.resolve( context, - Arrays.asList("testX*", "-doe", "-testXXX", "-testYYY") + resolvedExpressions("testX*", "-doe", "-testXXX", "-testYYY") ) ), - equalTo(newHashSet("testXYY")) + equalTo(resolvedExpressionsSet("testXYY")) ); if (indicesOptions == IndicesOptions.lenientExpandOpen()) { assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testXXX", "-testXXX"))), - equalTo(newHashSet("testXXX", "-testXXX")) + newHashSet( + IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testXXX", "-testXXX")) + ), + equalTo(resolvedExpressionsSet("testXXX", "-testXXX")) ); } else if (indicesOptions == IndicesOptions.strictExpandOpen()) { IndexNotFoundException infe = expectThrows( @@ -103,8 +106,8 @@ public void testConvertWildcardsJustIndicesTests() { assertEquals("-testXXX", infe.getIndex().getName()); } assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testXXX", "-testX*"))), - equalTo(newHashSet("testXXX")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testXXX", "-testX*"))), + equalTo(resolvedExpressionsSet("testXXX")) ); } @@ -122,24 +125,24 @@ public void testConvertWildcardsTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testYY*", "alias*"))), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testYY*", "alias*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("-kuku"))), - equalTo(newHashSet("-kuku")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("-kuku"))), + equalTo(resolvedExpressionsSet("-kuku")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("test*", "-testYYY"))), - equalTo(newHashSet("testXXX", "testXYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("test*", "-testYYY"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testX*", "testYYY"))), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testX*", "testYYY"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testYYY", "testX*"))), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testYYY", "testX*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); } @@ -159,8 +162,8 @@ public void testConvertWildcardsOpenClosedIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("testX*"))), - equalTo(newHashSet("testXXX", "testXXY", "testXYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testX*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); context = new IndexNameExpressionResolver.Context( state, @@ -168,8 +171,8 @@ public void testConvertWildcardsOpenClosedIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("testX*"))), - equalTo(newHashSet("testXYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testX*"))), + equalTo(resolvedExpressionsSet("testXYY")) ); context = new IndexNameExpressionResolver.Context( state, @@ -177,8 +180,8 @@ public void testConvertWildcardsOpenClosedIndicesTests() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("testX*"))), - equalTo(newHashSet("testXXX", "testXXY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("testX*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXXY")) ); context = new IndexNameExpressionResolver.Context( state, @@ -217,28 +220,27 @@ public void testMultipleWildcards() { SystemIndexAccessLevel.NONE ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("test*X*"))), - equalTo(newHashSet("testXXX", "testXXY", "testXYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("test*X*"))), + equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("test*X*Y"))), - equalTo(newHashSet("testXXY", "testXYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("test*X*Y"))), + equalTo(resolvedExpressionsSet("testXXY", "testXYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("kuku*Y*"))), - equalTo(newHashSet("kukuYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("kuku*Y*"))), + equalTo(resolvedExpressionsSet("kukuYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("*Y*"))), - equalTo(newHashSet("testXXY", "testXYY", "testYYY", "kukuYYY")) + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("*Y*"))), + equalTo(resolvedExpressionsSet("testXXY", "testXYY", "testYYY", "kukuYYY")) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("test*Y*X"))) - .size(), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("test*Y*X"))).size(), equalTo(0) ); assertThat( - newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("*Y*X"))).size(), + newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, resolvedExpressions("*Y*X"))).size(), equalTo(0) ); } @@ -257,11 +259,11 @@ public void testAll() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( newHashSet(IndexNameExpressionResolver.resolveExpressions(context, "_all")), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); IndicesOptions noExpandOptions = IndicesOptions.fromOptions( randomBoolean(), @@ -298,7 +300,7 @@ public void testAllAliases() { IndicesOptions.lenientExpandOpen(), // don't include hidden SystemIndexAccessLevel.NONE ); - assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet())); + assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(Set.of())); } { @@ -319,7 +321,7 @@ public void testAllAliases() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), - equalTo(newHashSet("index-visible-alias")) + equalTo(resolvedExpressionsSet("index-visible-alias")) ); } } @@ -362,7 +364,7 @@ public void testAllDataStreams() { assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), - equalTo(newHashSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis))) + equalTo(resolvedExpressionsSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis))) ); } @@ -385,7 +387,7 @@ public void testAllDataStreams() { NONE ); - assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet())); + assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(Set.of())); } } @@ -506,16 +508,16 @@ public void testResolveAliases() { ); { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAndAliasesContext, - Collections.singletonList("foo_a*") + resolvedExpressions("foo_a*") ); - assertThat(indices, containsInAnyOrder("foo_index", "bar_index")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_index", "bar_index"))); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( skipAliasesLenientContext, - Collections.singletonList("foo_a*") + resolvedExpressions("foo_a*") ); assertEquals(0, indices.size()); } @@ -524,45 +526,45 @@ public void testResolveAliases() { IndexNotFoundException.class, () -> IndexNameExpressionResolver.WildcardExpressionResolver.resolve( skipAliasesStrictContext, - Collections.singletonList("foo_a*") + resolvedExpressions("foo_a*") ) ); assertEquals("foo_a*", infe.getIndex().getName()); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAndAliasesContext, - Collections.singletonList("foo*") + resolvedExpressions("foo*") ); - assertThat(indices, containsInAnyOrder("foo_foo", "foo_index", "bar_index")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_foo", "foo_index", "bar_index"))); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( skipAliasesLenientContext, - Collections.singletonList("foo*") + resolvedExpressions("foo*") ); - assertThat(indices, containsInAnyOrder("foo_foo", "foo_index")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_foo", "foo_index"))); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( skipAliasesStrictContext, - Collections.singletonList("foo*") + resolvedExpressions("foo*") ); - assertThat(indices, containsInAnyOrder("foo_foo", "foo_index")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_foo", "foo_index"))); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAndAliasesContext, - Collections.singletonList("foo_alias") + resolvedExpressions("foo_alias") ); - assertThat(indices, containsInAnyOrder("foo_alias")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_alias"))); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( skipAliasesLenientContext, - Collections.singletonList("foo_alias") + resolvedExpressions("foo_alias") ); - assertThat(indices, containsInAnyOrder("foo_alias")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_alias"))); } { IllegalArgumentException iae = expectThrows( @@ -581,11 +583,11 @@ public void testResolveAliases() { SystemIndexAccessLevel.NONE ); { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( noExpandNoAliasesContext, - List.of("foo_alias") + resolvedExpressions("foo_alias") ); - assertThat(indices, containsInAnyOrder("foo_alias")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_alias"))); } IndicesOptions strictNoExpandNoAliasesIndicesOptions = IndicesOptions.fromOptions( false, @@ -654,18 +656,18 @@ public void testResolveDataStreams() { ); // data streams are not included but expression matches the data stream - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAndAliasesContext, - Collections.singletonList("foo_*") + resolvedExpressions("foo_*") ); - assertThat(indices, containsInAnyOrder("foo_index", "foo_foo", "bar_index")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("foo_index", "foo_foo", "bar_index"))); // data streams are not included and expression doesn't match the data steram indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAndAliasesContext, - Collections.singletonList("bar_*") + resolvedExpressions("bar_*") ); - assertThat(indices, containsInAnyOrder("bar_bar", "bar_index")); + assertThat(newHashSet(indices), equalTo(resolvedExpressionsSet("bar_bar", "bar_index"))); } { @@ -691,35 +693,39 @@ public void testResolveDataStreams() { ); // data stream's corresponding backing indices are resolved - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAliasesAndDataStreamsContext, - Collections.singletonList("foo_*") + resolvedExpressions("foo_*") ); assertThat( - indices, - containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + newHashSet(indices), + equalTo( + resolvedExpressionsSet( + "foo_index", + "bar_index", + "foo_foo", + DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), + DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + ) ) ); // include all wildcard adds the data stream's backing indices indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAliasesAndDataStreamsContext, - Collections.singletonList("*") + resolvedExpressions("*") ); assertThat( - indices, - containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - "bar_bar", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + newHashSet(indices), + equalTo( + resolvedExpressionsSet( + "foo_index", + "bar_index", + "foo_foo", + "bar_bar", + DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), + DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + ) ) ); } @@ -748,35 +754,39 @@ public void testResolveDataStreams() { ); // data stream's corresponding backing indices are resolved - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAliasesDataStreamsAndHiddenIndices, - Collections.singletonList("foo_*") + resolvedExpressions("foo_*") ); assertThat( - indices, - containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + newHashSet(indices), + equalTo( + resolvedExpressionsSet( + "foo_index", + "bar_index", + "foo_foo", + DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), + DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + ) ) ); // include all wildcard adds the data stream's backing indices indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( indicesAliasesDataStreamsAndHiddenIndices, - Collections.singletonList("*") + resolvedExpressions("*") ); assertThat( - indices, - containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - "bar_bar", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + newHashSet(indices), + equalTo( + resolvedExpressionsSet( + "foo_index", + "bar_index", + "foo_foo", + "bar_bar", + DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), + DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + ) ) ); } @@ -808,16 +818,28 @@ public void testMatchesConcreteIndicesWildcardAndAliases() { SystemIndexAccessLevel.NONE ); - Collection matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve(indicesAndAliasesContext, List.of("*")); - assertThat(matches, containsInAnyOrder("bar_bar", "foo_foo", "foo_index", "bar_index")); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve(onlyIndicesContext, List.of("*")); - assertThat(matches, containsInAnyOrder("bar_bar", "foo_foo", "foo_index", "bar_index")); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve(indicesAndAliasesContext, List.of("foo*")); - assertThat(matches, containsInAnyOrder("foo_foo", "foo_index", "bar_index")); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve(onlyIndicesContext, List.of("foo*")); - assertThat(matches, containsInAnyOrder("foo_foo", "foo_index")); - matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve(indicesAndAliasesContext, List.of("foo_alias")); - assertThat(matches, containsInAnyOrder("foo_alias")); + Collection matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + indicesAndAliasesContext, + List.of(new ResolvedExpression("*")) + ); + assertThat(newHashSet(matches), equalTo(resolvedExpressionsSet("bar_bar", "foo_foo", "foo_index", "bar_index"))); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve(onlyIndicesContext, List.of(new ResolvedExpression("*"))); + assertThat(newHashSet(matches), equalTo(resolvedExpressionsSet("bar_bar", "foo_foo", "foo_index", "bar_index"))); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + indicesAndAliasesContext, + List.of(new ResolvedExpression("foo*")) + ); + assertThat(newHashSet(matches), equalTo(resolvedExpressionsSet("foo_foo", "foo_index", "bar_index"))); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + onlyIndicesContext, + List.of(new ResolvedExpression("foo*")) + ); + assertThat(newHashSet(matches), equalTo(resolvedExpressionsSet("foo_foo", "foo_index"))); + matches = IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + indicesAndAliasesContext, + List.of(new ResolvedExpression("foo_alias")) + ); + assertThat(newHashSet(matches), equalTo(resolvedExpressionsSet("foo_alias"))); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> IndexNameExpressionResolver.resolveExpressions(onlyIndicesContext, "foo_alias") @@ -840,8 +862,19 @@ private static IndexMetadata.Builder indexBuilder(String index) { private static void assertWildcardResolvesToEmpty(IndexNameExpressionResolver.Context context, String wildcardExpression) { IndexNotFoundException infe = expectThrows( IndexNotFoundException.class, - () -> IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, List.of(wildcardExpression)) + () -> IndexNameExpressionResolver.WildcardExpressionResolver.resolve( + context, + List.of(new ResolvedExpression(wildcardExpression)) + ) ); assertEquals(wildcardExpression, infe.getIndex().getName()); } + + private List resolvedExpressions(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).toList(); + } + + private Set resolvedExpressionsSet(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).collect(Collectors.toSet()); + } } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java index 2793e03fc3fa8..b3af430cc43e2 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -390,28 +390,31 @@ public void testText() throws Exception { } public void testDate() throws Exception { - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (Date) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((Date) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (Date) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((Date) null).endObject()); final Date d1 = Date.from(ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timeValue(d1).endObject()); + assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timestampField("d1", d1).endObject()); + assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timestampValue(d1).endObject()); final Date d2 = Date.from(ZonedDateTime.of(2016, 12, 25, 7, 59, 42, 213000000, ZoneOffset.UTC).toInstant()); - assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().timeField("d2", d2).endObject()); - assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().field("d2").timeValue(d2).endObject()); + assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().timestampField("d2", d2).endObject()); + assertResult("{'d2':'2016-12-25T07:59:42.213Z'}", () -> builder().startObject().field("d2").timestampValue(d2).endObject()); } - public void testDateField() throws Exception { + public void testUnixEpochMillisField() throws Exception { final Date d = Date.from(ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant()); assertResult( "{'date_in_millis':1451606400000}", - () -> builder().startObject().timeField("date_in_millis", "date", d.getTime()).endObject() + () -> builder().startObject().timestampFieldsFromUnixEpochMillis("date_in_millis", "date", d.getTime()).endObject() ); assertResult( "{'date':'2016-01-01T00:00:00.000Z','date_in_millis':1451606400000}", - () -> builder().humanReadable(true).startObject().timeField("date_in_millis", "date", d.getTime()).endObject() + () -> builder().humanReadable(true) + .startObject() + .timestampFieldsFromUnixEpochMillis("date_in_millis", "date", d.getTime()) + .endObject() ); } @@ -419,7 +422,7 @@ public void testCalendar() throws Exception { Calendar calendar = GregorianCalendar.from(ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)); assertResult( "{'calendar':'2016-01-01T00:00:00.000Z'}", - () -> builder().startObject().field("calendar").timeValue(calendar).endObject() + () -> builder().startObject().field("calendar").timestampValue(calendar).endObject() ); } @@ -427,83 +430,95 @@ public void testJavaTime() throws Exception { final ZonedDateTime d1 = ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); // ZonedDateTime - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (ZonedDateTime) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((ZonedDateTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (ZonedDateTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((ZonedDateTime) null).endObject()); assertResult("{'date':null}", () -> builder().startObject().field("date", (ZonedDateTime) null).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timeValue(d1).endObject()); + assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timestampField("d1", d1).endObject()); + assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timestampValue(d1).endObject()); assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1).endObject()); // Instant - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (java.time.Instant) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((java.time.Instant) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (java.time.Instant) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((java.time.Instant) null).endObject()); assertResult("{'date':null}", () -> builder().startObject().field("date", (java.time.Instant) null).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1.toInstant()).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1").timeValue(d1.toInstant()).endObject()); + assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timestampField("d1", d1.toInstant()).endObject()); + assertResult( + "{'d1':'2016-01-01T00:00:00.000Z'}", + () -> builder().startObject().field("d1").timestampValue(d1.toInstant()).endObject() + ); assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1.toInstant()).endObject()); // LocalDateTime (no time zone) - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (LocalDateTime) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((LocalDateTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (LocalDateTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((LocalDateTime) null).endObject()); assertResult("{'date':null}", () -> builder().startObject().field("date", (LocalDateTime) null).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1.toLocalDateTime()).endObject()); assertResult( "{'d1':'2016-01-01T00:00:00.000Z'}", - () -> builder().startObject().field("d1").timeValue(d1.toLocalDateTime()).endObject() + () -> builder().startObject().timestampField("d1", d1.toLocalDateTime()).endObject() + ); + assertResult( + "{'d1':'2016-01-01T00:00:00.000Z'}", + () -> builder().startObject().field("d1").timestampValue(d1.toLocalDateTime()).endObject() ); assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1.toLocalDateTime()).endObject()); // LocalDate (no time, no time zone) - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (LocalDate) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((LocalDate) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (LocalDate) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((LocalDate) null).endObject()); assertResult("{'date':null}", () -> builder().startObject().field("date", (LocalDate) null).endObject()); - assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().timeField("d1", d1.toLocalDate()).endObject()); - assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().field("d1").timeValue(d1.toLocalDate()).endObject()); + assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().timestampField("d1", d1.toLocalDate()).endObject()); + assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().field("d1").timestampValue(d1.toLocalDate()).endObject()); assertResult("{'d1':'2016-01-01'}", () -> builder().startObject().field("d1", d1.toLocalDate()).endObject()); // LocalTime (no date, no time zone) - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (LocalTime) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((LocalTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (LocalTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((LocalTime) null).endObject()); assertResult("{'date':null}", () -> builder().startObject().field("date", (LocalTime) null).endObject()); - assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().timeField("d1", d1.toLocalTime()).endObject()); - assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().field("d1").timeValue(d1.toLocalTime()).endObject()); + assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().timestampField("d1", d1.toLocalTime()).endObject()); + assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().field("d1").timestampValue(d1.toLocalTime()).endObject()); assertResult("{'d1':'00:00:00.000'}", () -> builder().startObject().field("d1", d1.toLocalTime()).endObject()); final ZonedDateTime d2 = ZonedDateTime.of(2016, 1, 1, 7, 59, 23, 123_000_000, ZoneOffset.UTC); - assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().timeField("d1", d2.toLocalTime()).endObject()); - assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().field("d1").timeValue(d2.toLocalTime()).endObject()); + assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().timestampField("d1", d2.toLocalTime()).endObject()); + assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().field("d1").timestampValue(d2.toLocalTime()).endObject()); assertResult("{'d1':'07:59:23.123'}", () -> builder().startObject().field("d1", d2.toLocalTime()).endObject()); // OffsetDateTime - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (OffsetDateTime) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((OffsetDateTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (OffsetDateTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((OffsetDateTime) null).endObject()); assertResult("{'date':null}", () -> builder().startObject().field("date", (OffsetDateTime) null).endObject()); assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().field("d1", d1.toOffsetDateTime()).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000Z'}", () -> builder().startObject().timeField("d1", d1.toOffsetDateTime()).endObject()); assertResult( "{'d1':'2016-01-01T00:00:00.000Z'}", - () -> builder().startObject().field("d1").timeValue(d1.toOffsetDateTime()).endObject() + () -> builder().startObject().timestampField("d1", d1.toOffsetDateTime()).endObject() + ); + assertResult( + "{'d1':'2016-01-01T00:00:00.000Z'}", + () -> builder().startObject().field("d1").timestampValue(d1.toOffsetDateTime()).endObject() ); // also test with a date that has a real offset OffsetDateTime offsetDateTime = d1.withZoneSameLocal(ZoneOffset.ofHours(5)).toOffsetDateTime(); assertResult("{'d1':'2016-01-01T00:00:00.000+05:00'}", () -> builder().startObject().field("d1", offsetDateTime).endObject()); - assertResult("{'d1':'2016-01-01T00:00:00.000+05:00'}", () -> builder().startObject().timeField("d1", offsetDateTime).endObject()); assertResult( "{'d1':'2016-01-01T00:00:00.000+05:00'}", - () -> builder().startObject().field("d1").timeValue(offsetDateTime).endObject() + () -> builder().startObject().timestampField("d1", offsetDateTime).endObject() + ); + assertResult( + "{'d1':'2016-01-01T00:00:00.000+05:00'}", + () -> builder().startObject().field("d1").timestampValue(offsetDateTime).endObject() ); // OffsetTime - assertResult("{'date':null}", () -> builder().startObject().timeField("date", (OffsetTime) null).endObject()); - assertResult("{'date':null}", () -> builder().startObject().field("date").timeValue((OffsetTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().timestampField("date", (OffsetTime) null).endObject()); + assertResult("{'date':null}", () -> builder().startObject().field("date").timestampValue((OffsetTime) null).endObject()); assertResult("{'date':null}", () -> builder().startObject().field("date", (OffsetTime) null).endObject()); final OffsetTime offsetTime = d2.toOffsetDateTime().toOffsetTime(); - assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().timeField("o", offsetTime).endObject()); - assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().field("o").timeValue(offsetTime).endObject()); + assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().timestampField("o", offsetTime).endObject()); + assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().field("o").timestampValue(offsetTime).endObject()); assertResult("{'o':'07:59:23.123Z'}", () -> builder().startObject().field("o", offsetTime).endObject()); // also test with a date that has a real offset final OffsetTime zonedOffsetTime = offsetTime.withOffsetSameLocal(ZoneOffset.ofHours(5)); - assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().timeField("o", zonedOffsetTime).endObject()); - assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().field("o").timeValue(zonedOffsetTime).endObject()); + assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().timestampField("o", zonedOffsetTime).endObject()); + assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().field("o").timestampValue(zonedOffsetTime).endObject()); assertResult("{'o':'07:59:23.123+05:00'}", () -> builder().startObject().field("o", zonedOffsetTime).endObject()); // DayOfWeek enum, not a real time value, but might be used in scripts diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java index a695fb6c45348..575382c7fb441 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java @@ -189,7 +189,7 @@ public void testDateTypesConversion() throws Exception { Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT); String expectedCalendar = XContentElasticsearchExtension.DEFAULT_FORMATTER.format(calendar.toInstant()); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - builder.startObject().timeField("date", date).endObject(); + builder.startObject().timestampField("date", date).endObject(); assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}")); builder = XContentFactory.contentBuilder(XContentType.JSON); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java index 387340c0a6f50..628de0b047bf5 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java @@ -79,7 +79,7 @@ static class TestIndexSettingsProvider implements IndexSettingProvider { public Settings getAdditionalIndexSettings( String indexName, String dataStreamName, - boolean isTimeSeries, + IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java index 4ac66a9f63a3f..04d4ef2079b99 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java @@ -1741,6 +1741,6 @@ public int dimension() { similarityFunction ); - assertEquals(132.30249f, scorer.score(0), 0.0001f); + assertEquals(129.64046f, scorer.score(0), 0.0001f); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 7f430cf676809..7e9a196faaa26 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -1377,6 +1377,7 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() { } public void testSubobjectsAutoFlatPaths() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createDynamicTemplateAutoSubobjects(); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.field("foo.metric.count", 10); @@ -1389,6 +1390,7 @@ public void testSubobjectsAutoFlatPaths() throws IOException { } public void testSubobjectsAutoStructuredPaths() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createDynamicTemplateAutoSubobjects(); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.startObject("foo"); @@ -1411,6 +1413,7 @@ public void testSubobjectsAutoStructuredPaths() throws IOException { } public void testSubobjectsAutoArrayOfObjects() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createDynamicTemplateAutoSubobjects(); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.startObject("foo"); @@ -1444,6 +1447,7 @@ public void testSubobjectsAutoArrayOfObjects() throws IOException { } public void testSubobjectAutoDynamicNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); DocumentMapper mapper = createDocumentMapper(topMapping(b -> { b.startArray("dynamic_templates"); { @@ -1482,6 +1486,7 @@ public void testSubobjectAutoDynamicNested() throws IOException { } public void testRootSubobjectAutoDynamicNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); DocumentMapper mapper = createDocumentMapper(topMapping(b -> { b.startArray("dynamic_templates"); { @@ -1515,6 +1520,7 @@ public void testRootSubobjectAutoDynamicNested() throws IOException { } public void testDynamicSubobjectsAutoDynamicFalse() throws Exception { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); // verify that we read the dynamic value properly from the parent mapper. DocumentParser#dynamicOrDefault splits the field // name where dots are found, but it does that only for the parent prefix e.g. metrics.service and not for the leaf suffix time.max DocumentMapper mapper = createDocumentMapper(topMapping(b -> { @@ -1578,6 +1584,7 @@ public void testDynamicSubobjectsAutoDynamicFalse() throws Exception { } public void testSubobjectsAutoWithInnerNestedFromDynamicTemplate() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); DocumentMapper mapper = createDocumentMapper(topMapping(b -> { b.startArray("dynamic_templates"); { @@ -2045,6 +2052,7 @@ public void testSubobjectsFalseFlattened() throws IOException { } public void testSubobjectsAutoFlattened() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); String mapping = """ { "_doc": { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 64eee39532c31..3b77015fde415 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -169,27 +169,29 @@ public void testMergeEnabledForIndexTemplates() throws IOException { assertEquals(ObjectMapper.Subobjects.ENABLED, objectMapper.subobjects()); assertTrue(objectMapper.sourceKeepMode().isEmpty()); - // Setting 'enabled' to true is allowed, and updates the mapping. - update = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject("object") - .field("type", "object") - .field("enabled", true) - .field("subobjects", "auto") - .field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true) - .endObject() - .endObject() - .endObject() - ); - mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE); - - objectMapper = mapper.mappers().objectMappers().get("object"); - assertNotNull(objectMapper); - assertTrue(objectMapper.isEnabled()); - assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects()); - assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE)); + if (ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()) { + // Setting 'enabled' to true is allowed, and updates the mapping. + update = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("object") + .field("type", "object") + .field("enabled", true) + .field("subobjects", "auto") + .field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true) + .endObject() + .endObject() + .endObject() + ); + mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE); + + objectMapper = mapper.mappers().objectMappers().get("object"); + assertNotNull(objectMapper); + assertTrue(objectMapper.isEnabled()); + assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects()); + assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE)); + } } public void testFieldReplacementForIndexTemplates() throws IOException { @@ -503,6 +505,7 @@ public void testSubobjectsCannotBeUpdatedOnRoot() throws IOException { } public void testSubobjectsAuto() throws Exception { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -532,6 +535,7 @@ public void testSubobjectsAuto() throws Exception { } public void testSubobjectsAutoWithInnerObject() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -565,6 +569,7 @@ public void testSubobjectsAutoWithInnerObject() throws IOException { } public void testSubobjectsAutoWithInnerNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -586,6 +591,7 @@ public void testSubobjectsAutoWithInnerNested() throws IOException { } public void testSubobjectsAutoRoot() throws Exception { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { b.startObject("metrics.service.time"); b.field("type", "long"); @@ -606,6 +612,7 @@ public void testSubobjectsAutoRoot() throws Exception { } public void testSubobjectsAutoRootWithInnerObject() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { b.startObject("metrics.service.time"); { @@ -626,6 +633,7 @@ public void testSubobjectsAutoRootWithInnerObject() throws IOException { } public void testSubobjectsAutoRootWithInnerNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { b.startObject("metrics.service"); b.field("type", "nested"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 04b9b05ecfe3a..cd7ff54ffc938 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -63,6 +63,7 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_FEATURE_FLAG; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -1227,13 +1228,18 @@ public void testInvalidParameters() { e.getMessage(), containsString("Failed to parse mapping: Mapping definition for [field] has unsupported parameters: [foo : {}]") ); - for (String quantizationKind : new String[] { "int4_hnsw", "int8_hnsw", "int8_flat", "int4_flat" }) { + List floatOnlyQuantizations = new ArrayList<>(Arrays.asList("int4_hnsw", "int8_hnsw", "int8_flat", "int4_flat")); + if (BBQ_FEATURE_FLAG.isEnabled()) { + floatOnlyQuantizations.add("bbq_hnsw"); + floatOnlyQuantizations.add("bbq_flat"); + } + for (String quantizationKind : floatOnlyQuantizations) { e = expectThrows( MapperParsingException.class, () -> createDocumentMapper( fieldMapping( b -> b.field("type", "dense_vector") - .field("dims", dims) + .field("dims", 64) .field("element_type", "byte") .field("similarity", "l2_norm") .field("index", true) @@ -1939,6 +1945,62 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException { assertEquals(expectedString, knnVectorsFormat.toString()); } + public void testKnnBBQHNSWVectorsFormat() throws IOException { + assumeTrue("BBQ vectors are not supported in the current version", BBQ_FEATURE_FLAG.isEnabled()); + final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); + final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); + final int dims = randomIntBetween(64, 4096); + MapperService mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "dense_vector"); + b.field("dims", dims); + b.field("index", true); + b.field("similarity", "dot_product"); + b.startObject("index_options"); + b.field("type", "bbq_hnsw"); + b.field("m", m); + b.field("ef_construction", efConstruction); + b.endObject(); + })); + CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); + Codec codec = codecService.codec("default"); + KnnVectorsFormat knnVectorsFormat; + if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) { + assertThat(codec, instanceOf(PerFieldMapperCodec.class)); + knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); + } else { + if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) { + codec = deduplicateFieldInfosCodec.delegate(); + } + assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); + knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); + } + String expectedString = "ES816HnswBinaryQuantizedVectorsFormat(name=ES816HnswBinaryQuantizedVectorsFormat, maxConn=" + + m + + ", beamWidth=" + + efConstruction + + ", flatVectorFormat=ES816BinaryQuantizedVectorsFormat(" + + "name=ES816BinaryQuantizedVectorsFormat, " + + "flatVectorScorer=ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())))"; + assertEquals(expectedString, knnVectorsFormat.toString()); + } + + public void testInvalidVectorDimensionsBBQ() { + assumeTrue("BBQ vectors are not supported in the current version", BBQ_FEATURE_FLAG.isEnabled()); + for (String quantizedFlatFormat : new String[] { "bbq_hnsw", "bbq_flat" }) { + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { + b.field("type", "dense_vector"); + b.field("dims", randomIntBetween(1, 63)); + b.field("element_type", "float"); + b.field("index", true); + b.field("similarity", "dot_product"); + b.startObject("index_options"); + b.field("type", quantizedFlatFormat); + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("does not support dimensions fewer than 64")); + } + } + public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException { final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); @@ -2022,24 +2084,27 @@ protected boolean supportsEmptyInputArray() { private static class DenseVectorSyntheticSourceSupport implements SyntheticSourceSupport { private final int dims = between(5, 1000); - private final ElementType elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT); + private final ElementType elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT); private final boolean indexed = randomBoolean(); private final boolean indexOptionsSet = indexed && randomBoolean(); @Override public SyntheticSourceExample example(int maxValues) throws IOException { - Object value = elementType == ElementType.BYTE - ? randomList(dims, dims, ESTestCase::randomByte) - : randomList(dims, dims, ESTestCase::randomFloat); + Object value = switch (elementType) { + case BYTE, BIT: + yield randomList(dims, dims, ESTestCase::randomByte); + case FLOAT: + yield randomList(dims, dims, ESTestCase::randomFloat); + }; return new SyntheticSourceExample(value, value, this::mapping); } private void mapping(XContentBuilder b) throws IOException { b.field("type", "dense_vector"); - b.field("dims", dims); - if (elementType == ElementType.BYTE || randomBoolean()) { + if (elementType == ElementType.BYTE || elementType == ElementType.BIT || randomBoolean()) { b.field("element_type", elementType.toString()); } + b.field("dims", elementType == ElementType.BIT ? dims * Byte.SIZE : dims); if (indexed) { b.field("index", true); b.field("similarity", "l2_norm"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 23864777db961..6433cf2f1c0d4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Set; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @@ -61,7 +62,9 @@ private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() { ), new DenseVectorFieldMapper.FlatIndexOptions(), new DenseVectorFieldMapper.Int8FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true))), - new DenseVectorFieldMapper.Int4FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true))) + new DenseVectorFieldMapper.Int4FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true))), + new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), + new DenseVectorFieldMapper.BBQFlatIndexOptions() ); } @@ -70,7 +73,7 @@ private DenseVectorFieldType createFloatFieldType() { "f", IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT, - 6, + BBQ_MIN_DIMS, indexed, VectorSimilarity.COSINE, indexed ? randomIndexOptionsAll() : null, @@ -147,7 +150,7 @@ public void testFetchSourceValue() throws IOException { public void testCreateNestedKnnQuery() { BitSetProducer producer = context -> null; - int dims = randomIntBetween(2, 2048); + int dims = randomIntBetween(BBQ_MIN_DIMS, 2048); if (dims % 2 != 0) { dims++; } @@ -197,7 +200,7 @@ public void testCreateNestedKnnQuery() { } public void testExactKnnQuery() { - int dims = randomIntBetween(2, 2048); + int dims = randomIntBetween(BBQ_MIN_DIMS, 2048); if (dims % 2 != 0) { dims++; } @@ -260,15 +263,19 @@ public void testFloatCreateKnnQuery() { "f", IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT, - 4, + BBQ_MIN_DIMS, true, VectorSimilarity.DOT_PRODUCT, randomIndexOptionsAll(), Collections.emptyMap() ); + float[] queryVector = new float[BBQ_MIN_DIMS]; + for (int i = 0; i < BBQ_MIN_DIMS; i++) { + queryVector[i] = i; + } e = expectThrows( IllegalArgumentException.class, - () -> dotProductField.createKnnQuery(VectorData.fromFloats(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }), 10, 10, null, null, null) + () -> dotProductField.createKnnQuery(VectorData.fromFloats(queryVector), 10, 10, null, null, null) ); assertThat(e.getMessage(), containsString("The [dot_product] similarity can only be used with unit-length vectors.")); @@ -276,7 +283,7 @@ public void testFloatCreateKnnQuery() { "f", IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT, - 4, + BBQ_MIN_DIMS, true, VectorSimilarity.COSINE, randomIndexOptionsAll(), @@ -284,7 +291,7 @@ public void testFloatCreateKnnQuery() { ); e = expectThrows( IllegalArgumentException.class, - () -> cosineField.createKnnQuery(VectorData.fromFloats(new float[] { 0.0f, 0.0f, 0.0f, 0.0f }), 10, 10, null, null, null) + () -> cosineField.createKnnQuery(VectorData.fromFloats(new float[BBQ_MIN_DIMS]), 10, 10, null, null, null) ); assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude.")); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 36f7355a541c1..17975b7d18dd8 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -77,6 +78,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -677,27 +679,27 @@ public void testBuildAliasFilter() { ); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { - AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-0")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-0", resolvedExpressions("test-alias-0")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-0")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-1", resolvedExpressions("test-alias-0")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-0", resolvedExpressions("test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-1")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "baz"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-1", resolvedExpressions("test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-1")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bax"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-0", "test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-0", resolvedExpressions("test-alias-0", "test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0", "test-alias-1")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -706,7 +708,7 @@ public void testBuildAliasFilter() { assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bar"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-0", "test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-1", resolvedExpressions("test-alias-0", "test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0", "test-alias-1")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -718,7 +720,7 @@ public void testBuildAliasFilter() { AliasFilter result = indicesService.buildAliasFilter( state, "test-0", - Set.of("test-alias-0", "test-alias-1", "test-alias-non-filtering") + resolvedExpressions("test-alias-0", "test-alias-1", "test-alias-non-filtering") ); assertThat(result.getAliases(), emptyArray()); assertThat(result.getQueryBuilder(), nullValue()); @@ -727,7 +729,7 @@ public void testBuildAliasFilter() { AliasFilter result = indicesService.buildAliasFilter( state, "test-1", - Set.of("test-alias-0", "test-alias-1", "test-alias-non-filtering") + resolvedExpressions("test-alias-0", "test-alias-1", "test-alias-non-filtering") ); assertThat(result.getAliases(), emptyArray()); assertThat(result.getQueryBuilder(), nullValue()); @@ -754,19 +756,19 @@ public void testBuildAliasFilterDataStreamAliases() { ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar"))); } { String index = backingIndex2.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "baz"))); } { String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo", "logs")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo", "logs")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -776,7 +778,7 @@ public void testBuildAliasFilterDataStreamAliases() { } { String index = backingIndex2.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo", "logs")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo", "logs")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -787,13 +789,13 @@ public void testBuildAliasFilterDataStreamAliases() { { // querying an unfiltered and a filtered alias for the same data stream should drop the filters String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs", "logs_bar")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo", "logs", "logs_bar")); assertThat(result, is(AliasFilter.EMPTY)); } { // similarly, querying the data stream name and a filtered alias should drop the filter String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs", dataStreamName1)); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs", dataStreamName1)); assertThat(result, is(AliasFilter.EMPTY)); } } @@ -846,4 +848,8 @@ public void testWithTempIndexServiceHandlesExistingIndex() throws Exception { return null; }); } + + private Set resolvedExpressions(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).collect(Collectors.toSet()); + } } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java index b1b6a11764120..239c90bdee2fd 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java @@ -48,9 +48,9 @@ public void testMatchedTermsCount() throws IOException { // Partial match assertAllDocs( - Set.of(new Term("field", "foo"), new Term("field", "baz")), + Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")), ScriptTermStats::matchedTermsCount, - Map.of("doc-1", equalTo(1), "doc-2", equalTo(1), "doc-3", equalTo(0)) + Map.of("doc-1", equalTo(2), "doc-2", equalTo(1), "doc-3", equalTo(0)) ); // Always returns 0 when no term is provided. @@ -211,12 +211,12 @@ public void testTermFreq() throws IOException { // With missing terms { assertAllDocs( - Set.of(new Term("field", "foo"), new Term("field", "baz")), + Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")), ScriptTermStats::termFreq, Map.ofEntries( - Map.entry("doc-1", equalTo(new StatsSummary(2, 1, 0, 1))), - Map.entry("doc-2", equalTo(new StatsSummary(2, 2, 0, 2))), - Map.entry("doc-3", equalTo(new StatsSummary(2, 0, 0, 0))) + Map.entry("doc-1", equalTo(new StatsSummary(3, 2, 0, 1))), + Map.entry("doc-2", equalTo(new StatsSummary(3, 2, 0, 2))), + Map.entry("doc-3", equalTo(new StatsSummary(3, 0, 0, 0))) ) ); } @@ -274,10 +274,10 @@ public void testTermPositions() throws IOException { // With missing terms { assertAllDocs( - Set.of(new Term("field", "foo"), new Term("field", "baz")), + Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")), ScriptTermStats::termPositions, Map.ofEntries( - Map.entry("doc-1", equalTo(new StatsSummary(1, 1, 1, 1))), + Map.entry("doc-1", equalTo(new StatsSummary(2, 4, 1, 3))), Map.entry("doc-2", equalTo(new StatsSummary(2, 3, 1, 2))), Map.entry("doc-3", equalTo(new StatsSummary())) ) @@ -311,7 +311,7 @@ private void withIndexSearcher(CheckedConsumer consu Document doc = new Document(); doc.add(new TextField("id", "doc-1", Field.Store.YES)); - doc.add(new TextField("field", "foo bar", Field.Store.YES)); + doc.add(new TextField("field", "foo bar qux", Field.Store.YES)); w.addDocument(doc); doc = new Document(); diff --git a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java index ca05c57b7d733..b295b78453f93 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java @@ -195,6 +195,23 @@ public void testRankDocsQueryEarlyTerminate() throws IOException { assertThat(col.totalHits.value, equalTo((long) topSize)); assertEqualTopDocs(col.scoreDocs, rankDocs); } + + { + // A single rank doc in the last segment + RankDoc[] singleRankDoc = new RankDoc[1]; + singleRankDoc[0] = rankDocs[rankDocs.length - 1]; + RankDocsQuery q = new RankDocsQuery( + reader, + singleRankDoc, + new Query[] { NumericDocValuesField.newSlowExactQuery("active", 1) }, + new String[1], + false + ); + var topDocsManager = new TopScoreDocCollectorManager(1, null, 0); + var col = searcher.search(q, topDocsManager); + assertThat(col.totalHits.value, lessThanOrEqualTo((long) (2 + rankDocs.length))); + assertEqualTopDocs(col.scoreDocs, singleRankDoc); + } } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 57a6d1e09c52d..c4e1c6c7a0681 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -80,6 +80,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -104,6 +106,7 @@ public class MockTransportService extends TransportService { private final Map> openConnections = new HashMap<>(); private final List onStopListeners = new CopyOnWriteArrayList<>(); + private final AtomicReference> onConnectionClosedCallback = new AtomicReference<>(); public static class TestPlugin extends Plugin { @Override @@ -788,6 +791,19 @@ public void openConnection(DiscoveryNode node, ConnectionProfile connectionProfi })); } + public void setOnConnectionClosedCallback(Consumer callback) { + onConnectionClosedCallback.set(callback); + } + + @Override + public void onConnectionClosed(Transport.Connection connection) { + final Consumer callback = onConnectionClosedCallback.get(); + if (callback != null) { + callback.accept(connection); + } + super.onConnectionClosed(connection); + } + public void addOnStopListener(Runnable listener) { onStopListeners.add(listener); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index aa72d3248812e..3000819066495 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -18,8 +18,10 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), + SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null), CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null), - INFERENCE_DEFAULT_ELSER("es.inference_default_elser_feature_flag_enabled=true", Version.fromString("8.16.0"), null); + INFERENCE_DEFAULT_ELSER("es.inference_default_elser_feature_flag_enabled=true", Version.fromString("8.16.0"), null), + ML_SCALE_FROM_ZERO("es.ml_scale_from_zero_feature_flag_enabled=true", Version.fromString("8.16.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index b4be0b33a464e..ef03fd0ba6f0e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -332,8 +333,10 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); + replicatedSettings.add(IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); + replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java index 2b01f4d7fa2a4..0d1a007db0d39 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java @@ -524,13 +524,13 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t if (licenseVersion == VERSION_START) { builder.field(Fields.SUBSCRIPTION_TYPE, subscriptionType); } - builder.timeField(Fields.ISSUE_DATE_IN_MILLIS, Fields.ISSUE_DATE, issueDate); + builder.timestampFieldsFromUnixEpochMillis(Fields.ISSUE_DATE_IN_MILLIS, Fields.ISSUE_DATE, issueDate); if (licenseVersion == VERSION_START) { builder.field(Fields.FEATURE, feature); } if (expiryDate != LicenseSettings.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS) { - builder.timeField(Fields.EXPIRY_DATE_IN_MILLIS, Fields.EXPIRY_DATE, expiryDate); + builder.timestampFieldsFromUnixEpochMillis(Fields.EXPIRY_DATE_IN_MILLIS, Fields.EXPIRY_DATE, expiryDate); } if (licenseVersion >= VERSION_ENTERPRISE) { @@ -551,7 +551,7 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t builder.humanReadable(previouslyHumanReadable); } if (licenseVersion >= VERSION_START_DATE) { - builder.timeField(Fields.START_DATE_IN_MILLIS, Fields.START_DATE, startDate); + builder.timestampFieldsFromUnixEpochMillis(Fields.START_DATE_IN_MILLIS, Fields.START_DATE, startDate); } return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java index 5ba0e584d63bb..2f9b125352e9c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java @@ -226,7 +226,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("status", status.label()); if (expiryDate != BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS) { - builder.timeField("expiry_date_in_millis", "expiry_date", expiryDate); + builder.timestampFieldsFromUnixEpochMillis("expiry_date_in_millis", "expiry_date", expiryDate); } return builder.endObject(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java index 9c679cd04c94d..33402671a2236 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java @@ -489,7 +489,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (managedByILM) { builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName); if (indexCreationDate != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( INDEX_CREATION_DATE_MILLIS_FIELD.getPreferredName(), INDEX_CREATION_DATE_FIELD.getPreferredName(), indexCreationDate @@ -500,26 +500,42 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws ); } if (lifecycleDate != null) { - builder.timeField(LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), LIFECYCLE_DATE_FIELD.getPreferredName(), lifecycleDate); + builder.timestampFieldsFromUnixEpochMillis( + LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), + LIFECYCLE_DATE_FIELD.getPreferredName(), + lifecycleDate + ); builder.field(AGE_FIELD.getPreferredName(), getAge(nowSupplier).toHumanReadableString(2)); } if (phase != null) { builder.field(PHASE_FIELD.getPreferredName(), phase); } if (phaseTime != null) { - builder.timeField(PHASE_TIME_MILLIS_FIELD.getPreferredName(), PHASE_TIME_FIELD.getPreferredName(), phaseTime); + builder.timestampFieldsFromUnixEpochMillis( + PHASE_TIME_MILLIS_FIELD.getPreferredName(), + PHASE_TIME_FIELD.getPreferredName(), + phaseTime + ); } if (action != null) { builder.field(ACTION_FIELD.getPreferredName(), action); } if (actionTime != null) { - builder.timeField(ACTION_TIME_MILLIS_FIELD.getPreferredName(), ACTION_TIME_FIELD.getPreferredName(), actionTime); + builder.timestampFieldsFromUnixEpochMillis( + ACTION_TIME_MILLIS_FIELD.getPreferredName(), + ACTION_TIME_FIELD.getPreferredName(), + actionTime + ); } if (step != null) { builder.field(STEP_FIELD.getPreferredName(), step); } if (stepTime != null) { - builder.timeField(STEP_TIME_MILLIS_FIELD.getPreferredName(), STEP_TIME_FIELD.getPreferredName(), stepTime); + builder.timestampFieldsFromUnixEpochMillis( + STEP_TIME_MILLIS_FIELD.getPreferredName(), + STEP_TIME_FIELD.getPreferredName(), + stepTime + ); } if (Strings.hasLength(failedStep)) { builder.field(FAILED_STEP_FIELD.getPreferredName(), failedStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java index 78ff08d5ced5b..2aed198d2e5fe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfo.java @@ -130,7 +130,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(PHASE_DEFINITION_FIELD.getPreferredName(), phase); } builder.field(VERSION_FIELD.getPreferredName(), version); - builder.timeField(MODIFIED_DATE_IN_MILLIS_FIELD.getPreferredName(), "modified_date", modifiedDate); + builder.timestampFieldsFromUnixEpochMillis(MODIFIED_DATE_IN_MILLIS_FIELD.getPreferredName(), "modified_date", modifiedDate); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java index 3e61f6b4e9258..6c49cadb8d189 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java @@ -37,6 +37,14 @@ public final class MachineLearningField { Setting.Property.NodeScope ); + public static final Setting MAX_LAZY_ML_NODES = Setting.intSetting( + "xpack.ml.max_lazy_ml_nodes", + 0, + 0, + Setting.Property.OperatorDynamic, + Setting.Property.NodeScope + ); + /** * This boolean value indicates if `max_machine_memory_percent` should be ignored and an automatic calculation is used instead. * diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java index 082f6d7aff899..72f05091c1ccd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java @@ -255,7 +255,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("flushed", flushed); if (lastFinalizedBucketEnd != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( FlushAcknowledgement.LAST_FINALIZED_BUCKET_END.getPreferredName(), FlushAcknowledgement.LAST_FINALIZED_BUCKET_END.getPreferredName() + "_string", lastFinalizedBucketEnd.toEpochMilli() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java index d4da74df85ba9..2ea605753ccfc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java @@ -332,17 +332,33 @@ public String getByFieldValue() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ANNOTATION.getPreferredName(), annotation); - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + CREATE_TIME.getPreferredName(), + CREATE_TIME.getPreferredName() + "_string", + createTime.getTime() + ); builder.field(CREATE_USERNAME.getPreferredName(), createUsername); - builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + TIMESTAMP.getPreferredName(), + TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); if (endTimestamp != null) { - builder.timeField(END_TIMESTAMP.getPreferredName(), END_TIMESTAMP.getPreferredName() + "_string", endTimestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + END_TIMESTAMP.getPreferredName(), + END_TIMESTAMP.getPreferredName() + "_string", + endTimestamp.getTime() + ); } if (jobId != null) { builder.field(Job.ID.getPreferredName(), jobId); } if (modifiedTime != null) { - builder.timeField(MODIFIED_TIME.getPreferredName(), MODIFIED_TIME.getPreferredName() + "_string", modifiedTime.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + MODIFIED_TIME.getPreferredName(), + MODIFIED_TIME.getPreferredName() + "_string", + modifiedTime.getTime() + ); } if (modifiedUsername != null) { builder.field(MODIFIED_USERNAME.getPreferredName(), modifiedUsername); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java index c6fa4e052c683..b007c1da451f5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java @@ -217,8 +217,16 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(DESCRIPTION.getPreferredName(), description); - builder.timeField(START_TIME.getPreferredName(), START_TIME.getPreferredName() + "_string", startTime.toEpochMilli()); - builder.timeField(END_TIME.getPreferredName(), END_TIME.getPreferredName() + "_string", endTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + START_TIME.getPreferredName(), + START_TIME.getPreferredName() + "_string", + startTime.toEpochMilli() + ); + builder.timestampFieldsFromUnixEpochMillis( + END_TIME.getPreferredName(), + END_TIME.getPreferredName() + "_string", + endTime.toEpochMilli() + ); builder.field(SKIP_RESULT.getPreferredName(), skipResult); builder.field(SKIP_MODEL_UPDATE.getPreferredName(), skipModelUpdate); if (forceTimeShift != null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java index 7a3334aad00f1..694d248efc7be 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/SearchInterval.java @@ -30,8 +30,8 @@ public SearchInterval(StreamInput in) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.timeField(START_MS.getPreferredName(), START.getPreferredName(), startMs); - builder.timeField(END_MS.getPreferredName(), END.getPreferredName(), endMs); + builder.timestampFieldsFromUnixEpochMillis(START_MS.getPreferredName(), START.getPreferredName(), startMs); + builder.timestampFieldsFromUnixEpochMillis(END_MS.getPreferredName(), END.getPreferredName(), endMs); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 4c9028f64c2fd..779c6ef263ebe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -258,7 +258,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(ID.getPreferredName(), id); if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + CREATE_TIME.getPreferredName(), + CREATE_TIME.getPreferredName() + "_string", + createTime.toEpochMilli() + ); } if (version != null) { builder.field(VERSION.getPreferredName(), version); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java index e61517569445b..61c18c7c84161 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java @@ -143,7 +143,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(REASON.getPreferredName(), reason); } if (lastStateChangeTime != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LAST_STATE_CHANGE_TIME.getPreferredName(), LAST_STATE_CHANGE_TIME.getPreferredName() + "_string", lastStateChangeTime.toEpochMilli() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java index 8b7cff0e80441..0bc191defa6ec 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStats.java @@ -131,7 +131,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE); builder.field(Fields.JOB_ID.getPreferredName(), jobId); } - builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + Fields.TIMESTAMP.getPreferredName(), + Fields.TIMESTAMP.getPreferredName() + "_string", + timestamp.toEpochMilli() + ); builder.field(ITERATION.getPreferredName(), iteration); builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters); builder.field(TIMING_STATS.getPreferredName(), timingStats); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java index 9e9ff3e759e49..c5941cefb1531 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsage.java @@ -126,7 +126,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.JOB_ID.getPreferredName(), jobId); } if (timestamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java index 6ddc078bef4af..b78b495015ab1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStats.java @@ -101,7 +101,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE); builder.field(Fields.JOB_ID.getPreferredName(), jobId); } - builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + Fields.TIMESTAMP.getPreferredName(), + Fields.TIMESTAMP.getPreferredName() + "_string", + timestamp.toEpochMilli() + ); builder.field(PARAMETERS.getPreferredName(), parameters); builder.field(TIMING_STATS.getPreferredName(), timingStats); builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java index 7fff20bcb68ee..c411f3e5353fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStats.java @@ -131,7 +131,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.TYPE.getPreferredName(), TYPE_VALUE); builder.field(Fields.JOB_ID.getPreferredName(), jobId); } - builder.timeField(Fields.TIMESTAMP.getPreferredName(), Fields.TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + Fields.TIMESTAMP.getPreferredName(), + Fields.TIMESTAMP.getPreferredName() + "_string", + timestamp.toEpochMilli() + ); builder.field(ITERATION.getPreferredName(), iteration); builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters); builder.field(TIMING_STATS.getPreferredName(), timingStats); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index f0909f75d9402..5ae19f6db6bb4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -509,7 +509,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(CREATED_BY.getPreferredName(), createdBy); builder.field(VERSION.getPreferredName(), version.toString()); - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + CREATE_TIME.getPreferredName(), + CREATE_TIME.getPreferredName() + "_string", + createTime.toEpochMilli() + ); // If we are NOT storing the model, we should return the deprecated field name if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false) == false && builder.getRestApiVersion().matches(RestApiVersion.equalTo(RestApiVersion.V_7))) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java index aadaa5254ff15..858d97bf6f956 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java @@ -297,7 +297,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("inference_cache_hit_count", cacheHitCount); } if (lastAccess != null) { - builder.timeField("last_access", "last_access_string", lastAccess.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis("last_access", "last_access_string", lastAccess.toEpochMilli()); } if (pendingCount != null) { builder.field("number_of_pending_requests", pendingCount); @@ -312,7 +312,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("timeout_count", timeoutCount); } if (startTime != null) { - builder.timeField("start_time", "start_time_string", startTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis("start_time", "start_time_string", startTime.toEpochMilli()); } if (threadsPerAllocation != null) { builder.field("threads_per_allocation", threadsPerAllocation); @@ -608,7 +608,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("cache_size", cacheSize); } builder.field("priority", priority); - builder.timeField("start_time", "start_time_string", startTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis("start_time", "start_time_string", startTime.toEpochMilli()); int totalErrorCount = nodeStats.stream().mapToInt(NodeStats::getErrorCount).sum(); int totalRejectedExecutionCount = nodeStats.stream().mapToInt(NodeStats::getRejectedExecutionCount).sum(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java index 4a87b8e24f481..06c3f75587d62 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java @@ -224,15 +224,12 @@ public boolean hasStartedRoutes() { return nodeRoutingTable.values().stream().anyMatch(routeInfo -> routeInfo.getState() == RoutingState.STARTED); } - public List> selectRandomStartedNodesWeighedOnAllocationsForNRequests( - int numberOfRequests, - RoutingState requiredState - ) { + public List> selectRandomNodesWeighedOnAllocations(int numberOfRequests, RoutingState... acceptableStates) { List nodeIds = new ArrayList<>(nodeRoutingTable.size()); List cumulativeAllocations = new ArrayList<>(nodeRoutingTable.size()); int allocationSum = 0; for (Map.Entry routingEntry : nodeRoutingTable.entrySet()) { - if (routingEntry.getValue().getState() == requiredState) { + if (routingEntry.getValue().getState().isAnyOf(acceptableStates)) { nodeIds.add(routingEntry.getKey()); allocationSum += routingEntry.getValue().getCurrentAllocations(); cumulativeAllocations.add(allocationSum); @@ -368,7 +365,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (reason != null) { builder.field(REASON.getPreferredName(), reason); } - builder.timeField(START_TIME.getPreferredName(), startTime); + builder.timestampField(START_TIME.getPreferredName(), startTime); builder.field(MAX_ASSIGNED_ALLOCATIONS.getPreferredName(), maxAssignedAllocations); builder.field(ADAPTIVE_ALLOCATIONS.getPreferredName(), adaptiveAllocationsSettings); builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java index 5314702be0688..1721ae0b21349 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java @@ -162,7 +162,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(INFERENCE_COUNT.getPreferredName(), inferenceCount); builder.field(CACHE_MISS_COUNT.getPreferredName(), cacheMissCount); builder.field(MISSING_ALL_FIELDS_COUNT.getPreferredName(), missingAllFieldsCount); - builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timeStamp.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + TIMESTAMP.getPreferredName(), + TIMESTAMP.getPreferredName() + "_string", + timeStamp.toEpochMilli() + ); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java index d921bc1d4a158..cfbc6c6701427 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfig.java @@ -260,7 +260,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(MINIMUM_VERSION.getPreferredName(), minimumVersion); } if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + CREATE_TIME.getPreferredName(), + CREATE_TIME.getPreferredName() + "_string", + createTime.toEpochMilli() + ); } if (size > 0) { builder.field(SIZE.getPreferredName(), size); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 8da0209e10293..e663bbd6800bd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -613,9 +613,13 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (jobVersion != null) { builder.field(JOB_VERSION.getPreferredName(), jobVersion); } - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + CREATE_TIME.getPreferredName(), + CREATE_TIME.getPreferredName() + humanReadableSuffix, + createTime.getTime() + ); if (finishedTime != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, finishedTime.getTime() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java index 2d03d4273013d..64c449020daa8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java @@ -150,7 +150,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(REASON.getPreferredName(), reason); } if (lastStateChangeTime != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LAST_STATE_CHANGE_TIME.getPreferredName(), LAST_STATE_CHANGE_TIME.getPreferredName() + "_string", lastStateChangeTime.toEpochMilli() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java index 2254959242eab..24a6668a0c016 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java @@ -99,7 +99,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(ID.getPreferredName(), id); if (lastFinalizedBucketEnd != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LAST_FINALIZED_BUCKET_END.getPreferredName(), LAST_FINALIZED_BUCKET_END.getPreferredName() + "_string", lastFinalizedBucketEnd.toEpochMilli() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java index 91f09bc8171da..fe8cb390db805 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStats.java @@ -195,8 +195,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DEAD_CATEGORY_COUNT_FIELD.getPreferredName(), deadCategoryCount); builder.field(FAILED_CATEGORY_COUNT_FIELD.getPreferredName(), failedCategoryCount); builder.field(CATEGORIZATION_STATUS_FIELD.getPreferredName(), categorizationStatus); - builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.toEpochMilli()); - builder.timeField(TIMESTAMP_FIELD.getPreferredName(), TIMESTAMP_FIELD.getPreferredName() + "_string", timestamp.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + LOG_TIME_FIELD.getPreferredName(), + LOG_TIME_FIELD.getPreferredName() + "_string", + logTime.toEpochMilli() + ); + builder.timestampFieldsFromUnixEpochMillis( + TIMESTAMP_FIELD.getPreferredName(), + TIMESTAMP_FIELD.getPreferredName() + "_string", + timestamp.toEpochMilli() + ); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java index 775640ac2048f..4c9a3a4b70ecb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java @@ -583,35 +583,35 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.field(SPARSE_BUCKET_COUNT.getPreferredName(), sparseBucketCount); builder.field(BUCKET_COUNT.getPreferredName(), bucketCount); if (earliestRecordTimeStamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( EARLIEST_RECORD_TIME.getPreferredName(), EARLIEST_RECORD_TIME.getPreferredName() + "_string", earliestRecordTimeStamp.getTime() ); } if (latestRecordTimeStamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LATEST_RECORD_TIME.getPreferredName(), LATEST_RECORD_TIME.getPreferredName() + "_string", latestRecordTimeStamp.getTime() ); } if (lastDataTimeStamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LAST_DATA_TIME.getPreferredName(), LAST_DATA_TIME.getPreferredName() + "_string", lastDataTimeStamp.getTime() ); } if (latestEmptyBucketTimeStamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LATEST_EMPTY_BUCKET_TIME.getPreferredName(), LATEST_EMPTY_BUCKET_TIME.getPreferredName() + "_string", latestEmptyBucketTimeStamp.getTime() ); } if (latestSparseBucketTimeStamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LATEST_SPARSE_BUCKET_TIME.getPreferredName(), LATEST_SPARSE_BUCKET_TIME.getPreferredName() + "_string", latestSparseBucketTimeStamp.getTime() @@ -619,7 +619,11 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th } builder.field(INPUT_RECORD_COUNT.getPreferredName(), getInputRecordCount()); if (logTime != null) { - builder.timeField(LOG_TIME.getPreferredName(), LOG_TIME.getPreferredName() + "_string", logTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + LOG_TIME.getPreferredName(), + LOG_TIME.getPreferredName() + "_string", + logTime.toEpochMilli() + ); } return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java index 16eceb1e89a95..a95ee13f57913 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java @@ -363,9 +363,17 @@ public XContentBuilder doXContentBody(XContentBuilder builder) throws IOExceptio builder.field(DEAD_CATEGORY_COUNT_FIELD.getPreferredName(), deadCategoryCount); builder.field(FAILED_CATEGORY_COUNT_FIELD.getPreferredName(), failedCategoryCount); builder.field(CATEGORIZATION_STATUS_FIELD.getPreferredName(), categorizationStatus); - builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + LOG_TIME_FIELD.getPreferredName(), + LOG_TIME_FIELD.getPreferredName() + "_string", + logTime.getTime() + ); if (timestamp != null) { - builder.timeField(TIMESTAMP_FIELD.getPreferredName(), TIMESTAMP_FIELD.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + TIMESTAMP_FIELD.getPreferredName(), + TIMESTAMP_FIELD.getPreferredName() + "_string", + timestamp.getTime() + ); } return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java index bf62a8a267f84..3114c03879eb7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java @@ -194,7 +194,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Job.ID.getPreferredName(), jobId); builder.field(MIN_VERSION.getPreferredName(), minVersion); if (timestamp != null) { - builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + TIMESTAMP.getPreferredName(), + TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); } if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); @@ -207,14 +211,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName(), modelSizeStats); } if (latestRecordTimeStamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LATEST_RECORD_TIME.getPreferredName(), LATEST_RECORD_TIME.getPreferredName() + "_string", latestRecordTimeStamp.getTime() ); } if (latestResultTimeStamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LATEST_RESULT_TIME.getPreferredName(), LATEST_RESULT_TIME.getPreferredName() + "_string", latestResultTimeStamp.getTime() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java index ca1fd98b7bfb3..3b4d5b6a72654 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java @@ -283,7 +283,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(Detector.DETECTOR_INDEX.getPreferredName(), detectorIndex); builder.field(Result.IS_INTERIM.getPreferredName(), isInterim); - builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); if (byFieldName != null) { builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java index b4798b404a434..f867511d992c6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java @@ -173,7 +173,11 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(JOB_ID.getPreferredName(), jobId); - builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java index f659ceced3565..131e0c24b387e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java @@ -132,7 +132,11 @@ XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws I builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); builder.field(RAW_ANOMALY_SCORE.getPreferredName(), rawAnomalyScore); builder.field(PROBABILITY.getPreferredName(), probability); - builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(Result.IS_INTERIM.getPreferredName(), isInterim); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java index 20a2fa95b08f3..37eba1fc081a0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java @@ -140,7 +140,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex); if (timestamp != null) { - builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); } if (partitionFieldName != null) { builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java index b544c43295bc5..930c8b6f3ef68 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java @@ -132,7 +132,11 @@ XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws I builder.field(PROBABILITY.getPreferredName(), probability); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(Result.IS_INTERIM.getPreferredName(), isInterim); - builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java index ba1a03c64e15e..043611f3333f6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java @@ -153,7 +153,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex); if (timestamp != null) { - builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); } if (partitionFieldName != null) { builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java index c04a61951ad99..8cdcaa0205b0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/OverallBucket.java @@ -71,7 +71,11 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.timestampFieldsFromUnixEpochMillis( + Result.TIMESTAMP.getPreferredName(), + Result.TIMESTAMP.getPreferredName() + "_string", + timestamp.getTime() + ); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(OVERALL_SCORE.getPreferredName(), overallScore); builder.field(JOBS.getPreferredName(), jobs); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java index 39d822b843d15..e102f0712b283 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContext.java @@ -178,7 +178,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(INCREMENTAL_METRIC_VALUE_MS.getPreferredName(), incrementalMetricValueMs); if (latestTimestamp != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( LATEST_TIMESTAMP.getPreferredName(), LATEST_TIMESTAMP.getPreferredName() + "_string", latestTimestamp.toEpochMilli() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index b632c680260cf..32b401ebfb32d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -238,12 +238,16 @@ public Iterator toXContentChunked(ToXContent.Params params } builder.field("is_partial", isPartial); builder.field("is_running", isRunning); - builder.timeField("start_time_in_millis", "start_time", startTimeMillis); - builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("expiration_time_in_millis", "expiration_time", expirationTimeMillis); if (searchResponse != null) { if (isRunning == false) { TimeValue took = searchResponse.getTook(); - builder.timeField("completion_time_in_millis", "completion_time", startTimeMillis + took.millis()); + builder.timestampFieldsFromUnixEpochMillis( + "completion_time_in_millis", + "completion_time", + startTimeMillis + took.millis() + ); } builder.field("response"); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java index 10b7730b58c9b..89d4be514adde 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncStatusResponse.java @@ -175,10 +175,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("id", id); builder.field("is_running", isRunning); builder.field("is_partial", isPartial); - builder.timeField("start_time_in_millis", "start_time", startTimeMillis); - builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("expiration_time_in_millis", "expiration_time", expirationTimeMillis); if (completionTimeMillis != null) { - builder.timeField("completion_time_in_millis", "completion_time", completionTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("completion_time_in_millis", "completion_time", completionTimeMillis); } RestActions.buildBroadcastShardsHeader(builder, params, totalShards, successfulShards, skippedShards, failedShards, null); if (clusters != null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java index 0ada92bbb1e68..186cd81537909 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotInvocationRecord.java @@ -106,9 +106,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { builder.field(SNAPSHOT_NAME.getPreferredName(), snapshotName); if (snapshotStartTimestamp != null) { - builder.timeField(START_TIMESTAMP.getPreferredName(), "start_time_string", snapshotStartTimestamp); + builder.timestampFieldsFromUnixEpochMillis(START_TIMESTAMP.getPreferredName(), "start_time_string", snapshotStartTimestamp); } - builder.timeField(TIMESTAMP.getPreferredName(), "time_string", snapshotFinishTimestamp); + builder.timestampFieldsFromUnixEpochMillis(TIMESTAMP.getPreferredName(), "time_string", snapshotFinishTimestamp); if (Objects.nonNull(details)) { builder.field(DETAILS.getPreferredName(), details); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java index c3c70e595eb75..ea52930f4ae84 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java @@ -157,7 +157,7 @@ public boolean equals(Object obj) { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(policy.getId()); builder.field(SnapshotLifecyclePolicyMetadata.VERSION.getPreferredName(), version); - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( SnapshotLifecyclePolicyMetadata.MODIFIED_DATE_MILLIS.getPreferredName(), SnapshotLifecyclePolicyMetadata.MODIFIED_DATE.getPreferredName(), modifiedDate @@ -169,7 +169,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (lastFailure != null) { builder.field(SnapshotLifecyclePolicyMetadata.LAST_FAILURE.getPreferredName(), lastFailure); } - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( SnapshotLifecyclePolicyMetadata.NEXT_EXECUTION_MILLIS.getPreferredName(), SnapshotLifecyclePolicyMetadata.NEXT_EXECUTION.getPreferredName(), policy.calculateNextExecution(modifiedDate, Clock.systemUTC()) @@ -249,7 +249,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(NAME.getPreferredName(), snapshotId.getName()); builder.field(UUID.getPreferredName(), snapshotId.getUUID()); builder.field(STATE.getPreferredName(), state); - builder.timeField(START_TIME.getPreferredName(), "start_time", startTime); + builder.timestampFieldsFromUnixEpochMillis(START_TIME.getPreferredName(), "start_time", startTime); if (failure != null) { builder.field(FAILURE.getPreferredName(), failure); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java index 672578787762e..dfaaa48f1e2cb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java @@ -192,7 +192,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(POLICY.getPreferredName(), policy); builder.field(HEADERS.getPreferredName(), headers); builder.field(VERSION.getPreferredName(), version); - builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate); + builder.timestampFieldsFromUnixEpochMillis(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate); if (Objects.nonNull(lastSuccess)) { builder.field(LAST_SUCCESS.getPreferredName(), lastSuccess); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java index ee077e5140606..06ff971ecf890 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java @@ -134,7 +134,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .field("subject_dn", subjectDn) .field("serial_number", serialNumber) .field("has_private_key", hasPrivateKey) - .timeField("expiry", expiry); + .timestampField("expiry", expiry); if (Strings.hasLength(issuer)) { builder.field("issuer", issuer); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java index 2828a46a28b8c..aa256940daa9b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStats.java @@ -93,14 +93,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(TransformField.CHECKPOINT_PROGRESS.getPreferredName(), checkpointProgress); } if (timestampMillis > 0) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( TransformField.TIMESTAMP_MILLIS.getPreferredName(), TransformField.TIMESTAMP.getPreferredName(), timestampMillis ); } if (timeUpperBoundMillis > 0) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( TransformField.TIME_UPPER_BOUND_MILLIS.getPreferredName(), TransformField.TIME_UPPER_BOUND.getPreferredName(), timeUpperBoundMillis diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java index c4530c535cbcf..a6e365b793d93 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfo.java @@ -217,10 +217,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(OPERATIONS_BEHIND, operationsBehind); } if (changesLastDetectedAt != null) { - builder.timeField(CHANGES_LAST_DETECTED_AT, CHANGES_LAST_DETECTED_AT_HUMAN, changesLastDetectedAt.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis( + CHANGES_LAST_DETECTED_AT, + CHANGES_LAST_DETECTED_AT_HUMAN, + changesLastDetectedAt.toEpochMilli() + ); } if (lastSearchTime != null) { - builder.timeField(LAST_SEARCH_TIME, LAST_SEARCH_TIME_HUMAN, lastSearchTime.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis(LAST_SEARCH_TIME, LAST_SEARCH_TIME_HUMAN, lastSearchTime.toEpochMilli()); } builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index fb782bdae0068..d8972dcf6a6be 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -450,7 +450,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(TransformField.VERSION.getPreferredName(), transformVersion); } if (createTime != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( TransformField.CREATE_TIME.getPreferredName(), TransformField.CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java index 5697e1793f0b0..451cfd89f31af 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java @@ -90,7 +90,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(COUNT, count); if (firstOccurrence != null) { - builder.timeField(FIRST_OCCURRENCE, FIRST_OCCURRENCE_HUMAN_READABLE, firstOccurrence.toEpochMilli()); + builder.timestampFieldsFromUnixEpochMillis(FIRST_OCCURRENCE, FIRST_OCCURRENCE_HUMAN_READABLE, firstOccurrence.toEpochMilli()); } return builder.endObject(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java index 4da5d46e82fa6..a7633ed0fa1a1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/QueuedWatch.java @@ -71,8 +71,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("watch_id", watchId); builder.field("watch_record_id", watchRecordId); - builder.timeField("triggered_time", triggeredTime); - builder.timeField("execution_time", executionTime); + builder.timestampField("triggered_time", triggeredTime); + builder.timestampField("execution_time", executionTime); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java index 2b80c32f3c327..49d0566dffeaa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionSnapshot.java @@ -108,8 +108,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("watch_id", watchId); builder.field("watch_record_id", watchRecordId); - builder.timeField("triggered_time", triggeredTime); - builder.timeField("execution_time", executionTime); + builder.timestampField("triggered_time", triggeredTime); + builder.timestampField("execution_time", executionTime); builder.field("execution_phase", phase); if (executedActions != null) { builder.array("executed_actions", executedActions); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java index 6a213d6f5e379..c3b6e0089b4ae 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java @@ -195,7 +195,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenNoS builder.addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STOPPED, "")); TrainedModelAssignment assignment = builder.build(); - assertThat(assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STARTED).isEmpty(), is(true)); + assertThat(assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STARTED).isEmpty(), is(true)); } public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenSingleStartedNode() { @@ -203,7 +203,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenSin builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTED, "")); TrainedModelAssignment assignment = builder.build(); - var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STARTED); + var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STARTED); assertThat(nodes, contains(new Tuple<>("node-1", 1))); } @@ -213,7 +213,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenASh builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTED, "")); TrainedModelAssignment assignment = builder.build(); - var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STOPPING); + var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STOPPING); assertThat(nodes, empty()); } @@ -223,7 +223,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenASh builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STOPPING, "")); TrainedModelAssignment assignment = builder.build(); - var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STOPPING); + var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STOPPING); assertThat(nodes, contains(new Tuple<>("node-1", 1))); } @@ -234,7 +234,7 @@ public void testSingleRequestWith2Nodes() { builder.addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STARTED, "")); TrainedModelAssignment assignment = builder.build(); - var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1, RoutingState.STARTED); + var nodes = assignment.selectRandomNodesWeighedOnAllocations(1, RoutingState.STARTED); assertThat(nodes, hasSize(1)); assertEquals(nodes.get(0).v2(), Integer.valueOf(1)); } @@ -248,7 +248,7 @@ public void testSelectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenMul final int selectionCount = 10000; final CountAccumulator countsPerNodeAccumulator = new CountAccumulator(); - var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(selectionCount, RoutingState.STARTED); + var nodes = assignment.selectRandomNodesWeighedOnAllocations(selectionCount, RoutingState.STARTED); assertThat(nodes, hasSize(3)); assertThat(nodes.stream().mapToInt(Tuple::v2).sum(), equalTo(selectionCount)); @@ -269,7 +269,7 @@ public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenMul builder.addRoutingEntry("node-3", new RoutingInfo(0, 0, RoutingState.STARTED, "")); TrainedModelAssignment assignment = builder.build(); final int selectionCount = 1000; - var nodeCounts = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(selectionCount, RoutingState.STARTED); + var nodeCounts = assignment.selectRandomNodesWeighedOnAllocations(selectionCount, RoutingState.STARTED); assertThat(nodeCounts, hasSize(3)); var selectedNodes = new HashSet(); diff --git a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java index 9e5f999d1f825..2b9d9b0875220 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java +++ b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java @@ -100,8 +100,9 @@ public List routes() { Route.builder(GET, "/_test_cluster/deprecated_settings") .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.current()) .build(), - // TODO: s/deprecated/deprecatedForRemoval when removing `deprecated` method - Route.builder(POST, "/_test_cluster/deprecated_settings").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.current()).build(), + Route.builder(POST, "/_test_cluster/deprecated_settings") + .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.current()) + .build(), Route.builder(GET, "/_test_cluster/compat_only") .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.minimumSupported()) .build(), diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml new file mode 100644 index 0000000000000..016d9f10fe77f --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/70_query_rule_test.yml @@ -0,0 +1,252 @@ +setup: + - requires: + cluster_features: [ "query_rules.test" ] + reason: Introduced in 8.16.0 + + - do: + query_rules.put_ruleset: + ruleset_id: test-ruleset + body: + rules: + - rule_id: rule1 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ search ] + actions: + ids: + - 'doc1' + - rule_id: rule2 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ ui ] + actions: + docs: + - '_index': 'test-index1' + '_id': 'doc2' + - rule_id: rule3 + type: pinned + criteria: + - type: contains + metadata: query_string + values: [ kibana, logstash ] + actions: + ids: + - 'doc2' + - 'doc3' + - rule_id: rule4 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ ops ] + actions: + ids: + - 'doc7' + - rule_id: rule5 + type: exclude + criteria: + - type: exact + metadata: query_string + values: [ search ] + actions: + ids: + - 'doc8' + +--- +teardown: + - do: + query_rules.delete_ruleset: + ruleset_id: test-ruleset + ignore: 404 + + - do: + query_rules.delete_ruleset: + ruleset_id: combined-ruleset + ignore: 404 + + - do: + query_rules.delete_ruleset: + ruleset_id: double-jeopardy-ruleset + ignore: 404 + +--- +"Test query rules, specifying a ruleset that does not exist": + - do: + catch: /resource_not_found_exception/ + query_rules.test: + ruleset_id: nonexistent-ruleset + body: + match_criteria: + foo: bar + + +--- +"Test query rules with an empty body": + - do: + catch: bad_request + query_rules.test: + ruleset_id: nonexistent-ruleset + body: { } + +--- +"Test query rules with an ID match": + + - do: + query_rules.test: + ruleset_id: test-ruleset + body: + match_criteria: + query_string: search + + - match: { total_matched_rules: 2 } + - match: { matched_rules.0.rule_id: 'rule1' } + - match: { matched_rules.1.rule_id: 'rule5' } + +--- +"As a user, test query rules with an ID match": + - skip: + features: headers + + - do: + catch: forbidden + headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + query_rules.test: + ruleset_id: test-ruleset + body: + match_criteria: + query_string: search + +--- +"Test query rules with a doc match": + + - do: + query_rules.test: + ruleset_id: test-ruleset + body: + match_criteria: + query_string: ui + + - match: { total_matched_rules: 1 } + - match: { matched_rules.0.rule_id: 'rule2' } + +--- +"As a user, test query rules with a doc match": + - skip: + features: headers + + - do: + catch: forbidden + headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + query_rules.test: + ruleset_id: test-ruleset + body: + match_criteria: + query_string: ui + +--- +"Test query rules with no matching rules": + + - do: + query_rules.test: + ruleset_id: test-ruleset + body: + match_criteria: + query_string: no-match + + - match: { total_matched_rules: 0 } + +--- +"Test rules where the same ID is both pinned and excluded": + - do: + query_rules.put_ruleset: + ruleset_id: double-jeopardy-ruleset + body: + rules: + - rule_id: rule1 + type: pinned + criteria: + - type: exact + metadata: foo + values: [ bar ] + actions: + ids: + - 'doc8' + - rule_id: rule2 + type: exclude + criteria: + - type: exact + metadata: foo + values: [ bar ] + actions: + ids: + - 'doc8' + + - do: + query_rules.test: + ruleset_id: double-jeopardy-ruleset + body: + match_criteria: + foo: bar + + - match: { total_matched_rules: 2 } + - match: { matched_rules.0.rule_id: 'rule1' } + - match: { matched_rules.1.rule_id: 'rule2' } + +--- +"Perform a rule query over a ruleset with combined numeric and text rule matching": + + - do: + query_rules.put_ruleset: + ruleset_id: combined-ruleset + body: + rules: + - rule_id: rule1 + type: pinned + criteria: + - type: exact + metadata: foo + values: [ bar ] + actions: + ids: + - 'doc1' + - rule_id: rule2 + type: pinned + criteria: + - type: lte + metadata: foo + values: [ 100 ] + actions: + ids: + - 'doc2' + - do: + query_rules.test: + ruleset_id: combined-ruleset + body: + match_criteria: + foo: 100 + + - match: { total_matched_rules: 1 } + - match: { matched_rules.0.rule_id: 'rule2' } + + - do: + query_rules.test: + ruleset_id: combined-ruleset + body: + match_criteria: + foo: bar + + - match: { total_matched_rules: 1 } + - match: { matched_rules.0.rule_id: 'rule1' } + + - do: + query_rules.test: + ruleset_id: combined-ruleset + body: + match_criteria: + foo: baz + + - match: { total_matched_rules: 0 } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index bdd4cae3dda81..d5aef3b8808e8 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -165,6 +165,8 @@ import org.elasticsearch.xpack.application.rules.action.RestListQueryRulesetsAction; import org.elasticsearch.xpack.application.rules.action.RestPutQueryRuleAction; import org.elasticsearch.xpack.application.rules.action.RestPutQueryRulesetAction; +import org.elasticsearch.xpack.application.rules.action.RestTestQueryRulesetAction; +import org.elasticsearch.xpack.application.rules.action.TestQueryRulesetAction; import org.elasticsearch.xpack.application.rules.action.TransportDeleteQueryRuleAction; import org.elasticsearch.xpack.application.rules.action.TransportDeleteQueryRulesetAction; import org.elasticsearch.xpack.application.rules.action.TransportGetQueryRuleAction; @@ -172,6 +174,7 @@ import org.elasticsearch.xpack.application.rules.action.TransportListQueryRulesetsAction; import org.elasticsearch.xpack.application.rules.action.TransportPutQueryRuleAction; import org.elasticsearch.xpack.application.rules.action.TransportPutQueryRulesetAction; +import org.elasticsearch.xpack.application.rules.action.TransportTestQueryRulesetAction; import org.elasticsearch.xpack.application.search.SearchApplicationIndexService; import org.elasticsearch.xpack.application.search.action.DeleteSearchApplicationAction; import org.elasticsearch.xpack.application.search.action.GetSearchApplicationAction; @@ -266,6 +269,7 @@ protected XPackLicenseState getLicenseState() { new ActionHandler<>(DeleteQueryRuleAction.INSTANCE, TransportDeleteQueryRuleAction.class), new ActionHandler<>(GetQueryRuleAction.INSTANCE, TransportGetQueryRuleAction.class), new ActionHandler<>(PutQueryRuleAction.INSTANCE, TransportPutQueryRuleAction.class), + new ActionHandler<>(TestQueryRulesetAction.INSTANCE, TransportTestQueryRulesetAction.class), usageAction, infoAction @@ -373,7 +377,8 @@ public List getRestHandlers( new RestPutQueryRulesetAction(getLicenseState()), new RestDeleteQueryRuleAction(getLicenseState()), new RestGetQueryRuleAction(getLicenseState()), - new RestPutQueryRuleAction(getLicenseState()) + new RestPutQueryRuleAction(getLicenseState()), + new RestTestQueryRulesetAction(getLicenseState()) ) ); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java index 81e072479d402..174bcbe886dfb 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java @@ -14,8 +14,17 @@ import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.application.rules.action.TestQueryRulesetAction.QUERY_RULES_TEST_API; public class EnterpriseSearchFeatures implements FeatureSpecification { + + @Override + public Set getFeatures() { + return Set.of(QUERY_RULES_TEST_API); + } + @Override public Map getHistoricalFeatures() { return Map.of( diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java index 0ecb35531ac09..c14bb8e9a4ec9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java @@ -331,12 +331,8 @@ public AppliedQueryRules applyRule(AppliedQueryRules appliedRules, Map identifyMatchingDocs(Map matchCriteria) { - List matchingDocs = new ArrayList<>(); + public boolean isRuleMatch(Map matchCriteria) { Boolean isRuleMatch = null; - - // All specified criteria in a rule must match for the rule to be applied for (QueryRuleCriteria criterion : criteria) { for (String match : matchCriteria.keySet()) { final Object matchValue = matchCriteria.get(match); @@ -349,8 +345,13 @@ private List identifyMatchingDocs(Map matchCr } } } + return isRuleMatch != null && isRuleMatch; + } - if (isRuleMatch != null && isRuleMatch) { + @SuppressWarnings("unchecked") + private List identifyMatchingDocs(Map matchCriteria) { + List matchingDocs = new ArrayList<>(); + if (isRuleMatch(matchCriteria)) { if (actions.containsKey(IDS_FIELD.getPreferredName())) { matchingDocs.addAll( ((List) actions.get(IDS_FIELD.getPreferredName())).stream().map(id -> new SpecifiedDocument(null, id)).toList() diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java index f7e6f166cf53f..1d5ba878264f7 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetAction.java @@ -31,7 +31,8 @@ public class GetQueryRulesetAction { - public static final String NAME = "cluster:admin/xpack/query_rules/get"; + public static final ActionType TYPE = new ActionType<>("cluster:admin/xpack/query_rules/get"); + public static final String NAME = TYPE.name(); public static final ActionType INSTANCE = new ActionType<>(NAME); private GetQueryRulesetAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetAction.java new file mode 100644 index 0000000000000..b6e02b3c37262 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetAction.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.EnterpriseSearchBaseRestHandler; +import org.elasticsearch.xpack.application.utils.LicenseUtils; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +@ServerlessScope(Scope.PUBLIC) +public class RestTestQueryRulesetAction extends EnterpriseSearchBaseRestHandler { + public RestTestQueryRulesetAction(XPackLicenseState licenseState) { + super(licenseState, LicenseUtils.Product.QUERY_RULES); + } + + @Override + public String getName() { + return "query_ruleset_test_action"; + } + + @Override + public List routes() { + return List.of(new Route(POST, "/" + EnterpriseSearch.QUERY_RULES_API_ENDPOINT + "/{ruleset_id}" + "/_test")); + } + + @Override + protected RestChannelConsumer innerPrepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + final String rulesetId = restRequest.param("ruleset_id"); + TestQueryRulesetAction.Request request = null; + if (restRequest.hasContent()) { + try (var parser = restRequest.contentParser()) { + request = TestQueryRulesetAction.Request.parse(parser, rulesetId); + } + } + final TestQueryRulesetAction.Request finalRequest = request; + return channel -> client.execute(TestQueryRulesetAction.INSTANCE, finalRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java new file mode 100644 index 0000000000000..28f4a3b38dd59 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetAction.java @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.rules.QueryRulesIndexService; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class TestQueryRulesetAction { + + public static final NodeFeature QUERY_RULES_TEST_API = new NodeFeature("query_rules.test"); + + // TODO - We'd like to transition this to require less stringent permissions + public static final ActionType TYPE = new ActionType<>("cluster:admin/xpack/query_rules/test"); + + public static final String NAME = TYPE.name(); + public static final ActionType INSTANCE = new ActionType<>(NAME); + + private TestQueryRulesetAction() {/* no instances */} + + public static class Request extends ActionRequest implements ToXContentObject, IndicesRequest { + private final String rulesetId; + private final Map matchCriteria; + + private static final ParseField RULESET_ID_FIELD = new ParseField("ruleset_id"); + private static final ParseField MATCH_CRITERIA_FIELD = new ParseField("match_criteria"); + + public Request(StreamInput in) throws IOException { + super(in); + this.rulesetId = in.readString(); + this.matchCriteria = in.readGenericMap(); + } + + public Request(String rulesetId, Map matchCriteria) { + this.rulesetId = rulesetId; + this.matchCriteria = matchCriteria; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(rulesetId)) { + validationException = addValidationError("ruleset_id missing", validationException); + } + + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(rulesetId); + out.writeGenericMap(matchCriteria); + } + + public String rulesetId() { + return rulesetId; + } + + public Map matchCriteria() { + return matchCriteria; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(rulesetId, request.rulesetId) && Objects.equals(matchCriteria, request.matchCriteria); + } + + @Override + public int hashCode() { + return Objects.hash(rulesetId, matchCriteria); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(RULESET_ID_FIELD.getPreferredName(), rulesetId); + builder.startObject(MATCH_CRITERIA_FIELD.getPreferredName()); + builder.mapContents(matchCriteria); + builder.endObject(); + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "test_query_ruleset_request", + false, + (p, name) -> { + @SuppressWarnings("unchecked") + Map matchCriteria = (Map) p[0]; + return new Request(name, matchCriteria); + } + + ); + static { + PARSER.declareObject(constructorArg(), (p, c) -> p.map(), MATCH_CRITERIA_FIELD); + PARSER.declareString(optionalConstructorArg(), RULESET_ID_FIELD); // Required for parsing + } + + public static Request parse(XContentParser parser, String name) { + return PARSER.apply(parser, name); + } + + @Override + public String[] indices() { + return new String[] { QueryRulesIndexService.QUERY_RULES_ALIAS_NAME }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.lenientExpandHidden(); + } + + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private final int totalMatchedRules; + private final List matchedRules; + + private static final ParseField TOTAL_MATCHED_RULES_FIELD = new ParseField("total_matched_rules"); + private static final ParseField MATCHED_RULES_FIELD = new ParseField("matched_rules"); + + public Response(StreamInput in) throws IOException { + super(in); + this.totalMatchedRules = in.readVInt(); + this.matchedRules = in.readCollectionAsList(MatchedRule::new); + } + + public Response(int totalMatchedRules, List matchedRules) { + this.totalMatchedRules = totalMatchedRules; + this.matchedRules = matchedRules; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(totalMatchedRules); + out.writeCollection(matchedRules, (stream, matchedRule) -> matchedRule.writeTo(stream)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TOTAL_MATCHED_RULES_FIELD.getPreferredName(), totalMatchedRules); + builder.startArray(MATCHED_RULES_FIELD.getPreferredName()); + for (MatchedRule matchedRule : matchedRules) { + builder.startObject(); + builder.field("ruleset_id", matchedRule.rulesetId()); + builder.field("rule_id", matchedRule.ruleId()); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(totalMatchedRules, response.totalMatchedRules) && Objects.equals(matchedRules, response.matchedRules); + } + + @Override + public int hashCode() { + return Objects.hash(totalMatchedRules, matchedRules); + } + } + + public record MatchedRule(String rulesetId, String ruleId) { + public MatchedRule(StreamInput in) throws IOException { + this(in.readString(), in.readString()); + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeString(rulesetId); + out.writeString(ruleId); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportTestQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportTestQueryRulesetAction.java new file mode 100644 index 0000000000000..115cdc516831a --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportTestQueryRulesetAction.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.rules.QueryRule; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +public class TransportTestQueryRulesetAction extends HandledTransportAction< + TestQueryRulesetAction.Request, + TestQueryRulesetAction.Response> { + + private final Client client; + + @Inject + public TransportTestQueryRulesetAction(TransportService transportService, ActionFilters actionFilters, Client client) { + super( + TestQueryRulesetAction.NAME, + transportService, + actionFilters, + TestQueryRulesetAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.client = client; + } + + @Override + protected void doExecute(Task task, TestQueryRulesetAction.Request request, ActionListener listener) { + GetQueryRulesetAction.Request getQueryRulesetRequest = new GetQueryRulesetAction.Request(request.rulesetId()); + executeAsyncWithOrigin( + client, + ENT_SEARCH_ORIGIN, + GetQueryRulesetAction.TYPE, + getQueryRulesetRequest, + ActionListener.wrap(getQueryRulesetResponse -> { + List matchedRules = new ArrayList<>(); + for (QueryRule rule : getQueryRulesetResponse.queryRuleset().rules()) { + if (rule.isRuleMatch(request.matchCriteria())) { + matchedRules.add(new TestQueryRulesetAction.MatchedRule(request.rulesetId(), rule.id())); + } + } + listener.onResponse(new TestQueryRulesetAction.Response(matchedRules.size(), matchedRules)); + }, listener::onFailure) + ); + } + +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java index 06adb29e32691..190b3c3e53169 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchModuleTestUtils.java @@ -115,4 +115,8 @@ public static QueryRuleset randomQueryRuleset() { return new QueryRuleset(id, rules); } + public static Map randomMatchCriteria() { + return randomMap(1, 3, () -> Tuple.tuple(randomIdentifier(), randomAlphaOfLengthBetween(0, 10))); + } + } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetActionTests.java new file mode 100644 index 0000000000000..dc2869e3ff0be --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/RestTestQueryRulesetActionTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules.action; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.AbstractRestEnterpriseSearchActionTests; +import org.elasticsearch.xpack.application.EnterpriseSearchBaseRestHandler; +import org.elasticsearch.xpack.application.utils.LicenseUtils; + +import java.util.Map; + +public class RestTestQueryRulesetActionTests extends AbstractRestEnterpriseSearchActionTests { + public void testWithNonCompliantLicense() throws Exception { + checkLicenseForRequest( + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withMethod(RestRequest.Method.POST) + .withParams(Map.of("ruleset_id", "ruleset-id")) + .withContent(new BytesArray(""" + { + "match_criteria": { + "foo": "bar" + } + } + """), XContentType.JSON) + .build(), + LicenseUtils.Product.QUERY_RULES + ); + } + + public void testInvalidRequestWithNonCompliantLicense() throws Exception { + checkLicenseForRequest( + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withMethod(RestRequest.Method.POST) + .withParams(Map.of("invalid_param_name", "invalid_value")) + .withContent(new BytesArray("{}"), XContentType.JSON) + .build(), + LicenseUtils.Product.QUERY_RULES + ); + } + + @Override + protected EnterpriseSearchBaseRestHandler getRestAction(XPackLicenseState licenseState) { + return new RestTestQueryRulesetAction(licenseState); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..7041de1106b50 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; + +public class TestQueryRulesetActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { + + private final String RULESET_NAME = "my-ruleset"; + + @Override + protected Writeable.Reader instanceReader() { + return TestQueryRulesetAction.Request::new; + } + + @Override + protected TestQueryRulesetAction.Request createTestInstance() { + return new TestQueryRulesetAction.Request(RULESET_NAME, EnterpriseSearchModuleTestUtils.randomMatchCriteria()); + } + + @Override + protected TestQueryRulesetAction.Request mutateInstance(TestQueryRulesetAction.Request instance) { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected TestQueryRulesetAction.Request doParseInstance(XContentParser parser) throws IOException { + return TestQueryRulesetAction.Request.parse(parser, RULESET_NAME); + } + + @Override + protected TestQueryRulesetAction.Request mutateInstanceForVersion(TestQueryRulesetAction.Request instance, TransportVersion version) { + return instance; + } + + @Override + protected List bwcVersions() { + return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_TEST_API)).collect(Collectors.toList()); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java new file mode 100644 index 0000000000000..a6562fb7b52af --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; + +public class TestQueryRulesetActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< + TestQueryRulesetAction.Response> { + + @Override + protected Writeable.Reader instanceReader() { + return TestQueryRulesetAction.Response::new; + } + + @Override + protected TestQueryRulesetAction.Response mutateInstance(TestQueryRulesetAction.Response instance) { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected TestQueryRulesetAction.Response createTestInstance() { + int totalMatchedRules = randomIntBetween(0, 10); + List matchedRules = IntStream.range(0, totalMatchedRules) + .mapToObj(i -> new TestQueryRulesetAction.MatchedRule(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10))) + .toList(); + return new TestQueryRulesetAction.Response(totalMatchedRules, matchedRules); + } + + @Override + protected TestQueryRulesetAction.Response mutateInstanceForVersion(TestQueryRulesetAction.Response instance, TransportVersion version) { + return instance; + } + + @Override + protected List bwcVersions() { + return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_TEST_API)).collect(Collectors.toList()); + } +} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index ab05a71b0e1c6..b817ec17c7bda 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -155,18 +155,19 @@ public static TypeResolution isNotNullAndFoldable(Expression e, String operation return resolution; } - public static TypeResolution isNotFoldable(Expression e, String operationName, ParamOrdinal paramOrd) { - if (e.foldable()) { + public static TypeResolution isNotNull(Expression e, String operationName, ParamOrdinal paramOrd) { + if (e.dataType() == DataType.NULL) { return new TypeResolution( format( null, - "{}argument of [{}] must be a table column, found constant [{}]", + "{}argument of [{}] cannot be null, received [{}]", paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ", operationName, Expressions.name(e) ) ); } + return TypeResolution.TYPE_RESOLVED; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index c0092caeb9d5d..b23703c6d8b66 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -425,6 +425,10 @@ public static boolean isRepresentable(DataType t) { && t.isCounter() == false; } + public static boolean isCounter(DataType t) { + return t == COUNTER_DOUBLE || t == COUNTER_INTEGER || t == COUNTER_LONG; + } + public static boolean isSpatialPoint(DataType t) { return t == GEO_POINT || t == CARTESIAN_POINT; } @@ -437,6 +441,10 @@ public static boolean isSpatial(DataType t) { return t == GEO_POINT || t == CARTESIAN_POINT || t == GEO_SHAPE || t == CARTESIAN_SHAPE; } + public static boolean isSortable(DataType t) { + return false == (t == SOURCE || isCounter(t) || isSpatial(t)); + } + public String nameUpper() { return name; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java index 48b5fd1605edf..8bfcf4ca5c405 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java @@ -79,4 +79,19 @@ public static int mapSize(int size) { } return (int) (size / 0.75f + 1f); } + + @SafeVarargs + @SuppressWarnings("varargs") + public static List nullSafeList(T... entries) { + if (entries == null || entries.length == 0) { + return emptyList(); + } + List list = new ArrayList<>(entries.length); + for (T entry : entries) { + if (entry != null) { + list.add(entry); + } + } + return list; + } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java index e7b16072f4b66..a685687e8bfc6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java @@ -169,8 +169,8 @@ public DriverSleeps sleeps() { @Override public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat(ChunkedToXContentHelper.startObject(), Iterators.single((b, p) -> { - b.timeField("start_millis", "start", startMillis); - b.timeField("stop_millis", "stop", stopMillis); + b.timestampFieldsFromUnixEpochMillis("start_millis", "start", startMillis); + b.timestampFieldsFromUnixEpochMillis("stop_millis", "stop", stopMillis); b.field("took_nanos", tookNanos); if (b.humanReadable()) { b.field("took_time", TimeValue.timeValueNanos(tookNanos)); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java index 217a0b033bed4..01e9a73c4fb5f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java @@ -62,9 +62,9 @@ public boolean isStillSleeping() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("reason", reason); - builder.timeField("sleep_millis", "sleep", sleep); + builder.timestampFieldsFromUnixEpochMillis("sleep_millis", "sleep", sleep); if (wake > 0) { - builder.timeField("wake_millis", "wake", wake); + builder.timestampFieldsFromUnixEpochMillis("wake_millis", "wake", wake); } return builder.endObject(); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 36035c48f182c..237c6a9af197f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -367,7 +367,7 @@ date1:date | dd_ms:integer 2023-12-02T11:00:00.000Z | 1000 ; -evalDateDiffMonthAsWhole0Months +evalDateDiffMonthAsWhole0Months#[skip:-8.14.1, reason:omitting millis/timezone not allowed before 8.14] ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00:00:00") | EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to) @@ -378,7 +378,7 @@ ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00: ; -evalDateDiffMonthAsWhole1Month +evalDateDiffMonthAsWhole1Month#[skip:-8.14.1, reason:omitting millis/timezone not allowed before 8.14] ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-02-01T00:00:00") | EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec new file mode 100644 index 0000000000000..b0578aa1a4ed0 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec @@ -0,0 +1,199 @@ +############################################### +# Tests for Match function +# + +matchWithField +required_capability: match_function + +// tag::match-with-field[] +from books +| where match(author, "Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +// end::match-with-field[] + +// tag::match-with-field-result[] +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; +// end::match-with-field-result[] + +matchWithMultipleFunctions +required_capability: match_function + +from books +| where match(title, "Return") AND match(author, "Tolkien") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +matchWithQueryExpressions +required_capability: match_function + +from books +| where match(title, CONCAT("Return ", " King")) +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +matchAfterKeep +required_capability: match_function + +from books +| keep book_no, author +| where match(author, "Faulkner") +| sort book_no +| limit 5; + +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; + +matchAfterDrop +required_capability: match_function + +from books +| drop ratings, description, year, publisher, title, author.keyword +| where match(author, "Faulkner") +| keep book_no, author +| sort book_no +| limit 5; + +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; + +matchAfterEval +required_capability: match_function + +from books +| eval stars = to_long(ratings / 2.0) +| where match(author, "Faulkner") +| sort book_no +| keep book_no, author, stars +| limit 5; + +book_no:keyword | author:text | stars:long +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 3 +2713 | William Faulkner | 2 +2847 | Colleen Faulkner | 3 +2883 | William Faulkner | 2 +3293 | Danny Faulkner | 2 +; + +matchWithConjunction +required_capability: match_function + +from books +| where match(title, "Rings") and ratings > 4.6 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +matchWithFunctionPushedToLucene +required_capability: match_function + +from hosts +| where match(host, "beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") +| keep card, host, ip0, ip1; +ignoreOrder:true + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth1 |beta |127.0.0.1 |127.0.0.2 +; + +matchWithNonPushableConjunction +required_capability: match_function + +from books +| where match(title, "Rings") and length(title) > 75 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +; + +matchWithMultipleWhereClauses +required_capability: match_function + +from books +| where match(title, "rings") +| where match(title, "lord") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2675 | The Lord of the Rings - Boxed Set +2714 | Return of the King Being the Third Part of The Lord of the Rings +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +matchMultivaluedField +required_capability: match_function + +from employees +| where match(job_positions, "Tech Lead") and match(job_positions, "Reporting Analyst") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10004 | Chirstian | Koblick +10010 | Duangkaew | Piveteau +10011 | Mary | Sluis +10088 | Jungsoon | Syrzycki +10093 | Sailaja | Desikan +10097 | Remzi | Waschkowski +; + +testMultiValuedFieldWithConjunction +required_capability: match_function + +from employees +| where match(job_positions, "Data Scientist") and match(job_positions, "Support Engineer") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10043 | Yishay | Tzvieli +; + +testMatchAndQueryStringFunctions +required_capability: match_function +required_capability: qstr_function + +from employees +| where match(job_positions, "Data Scientist") and qstr("job_positions: (Support Engineer) and gender: F") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10041 | Uri | Lenart +10043 | Yishay | Tzvieli +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec index 2f6313925032e..6dc03d0debcfa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -49,20 +49,6 @@ book_no:keyword | title:text 7350 | Return of the Shadow ; -qstrWithDisjunction -required_capability: qstr_function - -from books -| where qstr("title:Return") or year > 2020 -| keep book_no, title; -ignoreOrder:true - -book_no:keyword | title:text -2714 | Return of the King Being the Third Part of The Lord of the Rings -6818 | Hadji Murad -7350 | Return of the Shadow -; - qstrWithConjunction required_capability: qstr_function @@ -88,17 +74,16 @@ card:keyword |host:keyword |ip0:ip |ip1:ip eth1 |beta |127.0.0.1 |127.0.0.2 ; -qstrWithFunctionNotPushedToLucene +qstrWithNonPushableConjunction required_capability: qstr_function from books -| where qstr("title: rings") and length(description) > 600 +| where qstr("title: Rings") and length(title) > 75 | keep book_no, title; ignoreOrder:true book_no:keyword | title:text -2675 | The Lord of the Rings - Boxed Set -2714 | Return of the King Being the Third Part of The Lord of the Rings +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings ; qstrWithMultipleWhereClauses @@ -114,3 +99,55 @@ book_no:keyword | title:text 4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings 7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) ; + + +matchMultivaluedTextField +required_capability: match_function + +from employees +| where qstr("job_positions: (Tech Lead) AND job_positions:(Reporting Analyst)") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10004 | Chirstian | Koblick +10010 | Duangkaew | Piveteau +10011 | Mary | Sluis +10088 | Jungsoon | Syrzycki +10093 | Sailaja | Desikan +10097 | Remzi | Waschkowski +; + +matchMultivaluedNumericField +required_capability: match_function + +from employees +| where qstr("salary_change: [14 TO *]") +| keep emp_no, first_name, last_name, salary_change; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword | salary_change:double +10003 | Parto | Bamford | [12.82, 14.68] +10015 | Guoxiang | Nooteboom | [12.4, 14.25] +10023 | Bojan | Montemayor | [0.8, 14.63] +10040 | Weiyi | Meriste | [-8.94, 1.92, 6.97, 14.74] +10061 | Tse | Herber | [-2.58, -0.95, 14.39] +10065 | Satosi | Awdeh | [-9.81, -1.47, 14.44] +10099 | Valter | Sullins | [-8.78, -3.98, 10.71, 14.26] +; + +testMultiValuedFieldWithConjunction +required_capability: match_function + +from employees +| where (qstr("job_positions: (Data Scientist) OR job_positions:(Support Engineer)")) and gender == "F" +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10023 | Bojan | Montemayor +10041 | Uri | Lenart +10044 | Mingsen | Casley +10053 | Sanjiv | Zschoche +10069 | Margareta | Bierman +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 4c40808a4ff96..b72c8bcb05ae9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -1203,7 +1203,7 @@ count:long | country:k 1 | Poland ; -airportsSortCityName +airportsSortCityName#[skip:-8.13.3, reason:fixed in 8.13] FROM airports | SORT abbrev | LIMIT 5 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 8a2e9b402fbca..496a747fd9c2b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2290,3 +2290,186 @@ from employees m:integer |a:double |x:integer 74999 |48249.0 |0 ; + + +statsWithFiltering +required_capability: per_agg_filtering +from employees +| stats max = max(salary), max_f = max(salary) where salary < 50000, max_a = max(salary) where salary > 100, + min = min(salary), min_f = min(salary) where salary > 50000, min_a = min(salary) where salary > 100 +; + +max:integer |max_f:integer |max_a:integer | min:integer | min_f:integer | min_a:integer +74999 |49818 |74999 | 25324 | 50064 | 25324 +; + +statsWithEverythingFiltered +required_capability: per_agg_filtering +from employees +| stats max = max(salary), max_a = max(salary) where salary < 100, + min = min(salary), min_a = min(salary) where salary > 99999 +; + +max:integer |max_a:integer|min:integer | min_a:integer +74999 |null |25324 | null +; + +statsWithNullFilter +required_capability: per_agg_filtering +from employees +| stats max = max(salary), max_a = max(salary) where null, + min = min(salary), min_a = min(salary) where to_string(null) == "abc" +; + +max:integer |max_a:integer|min:integer | min_a:integer +74999 |null |25324 | null +; + +statsWithBasicExpressionFiltered +required_capability: per_agg_filtering +from employees +| stats max = max(salary), max_f = max(salary) where salary < 50000, + min = min(salary), min_f = min(salary) where salary > 50000, + exp_p = max(salary) + 10000 where salary < 50000, + exp_m = min(salary) % 10000 where salary > 50000 +; + +max:integer |max_f:integer|min:integer | min_f:integer|exp_p:integer | exp_m:integer +74999 |49818 |25324 | 50064 |59818 | 64 +; + +statsWithExpressionOverFilters +required_capability: per_agg_filtering +from employees +| stats max = max(salary), max_f = max(salary) where salary < 50000, + min = min(salary), min_f = min(salary) where salary > 50000, + exp_gt = max(salary) - min(salary) where salary > 50000, + exp_lt = max(salary) - min(salary) where salary < 50000 + +; + +max:integer |max_f:integer | min:integer | min_f:integer |exp_gt:integer | exp_lt:integer +74999 |49818 | 25324 | 50064 |24935 | 24494 +; + + +statsWithExpressionOfExpressionsOverFilters +required_capability: per_agg_filtering +from employees +| stats max = max(salary + 1), max_f = max(salary + 2) where salary < 50000, + min = min(salary - 1), min_f = min(salary - 2) where salary > 50000, + exp_gt = max(salary + 3) - min(salary - 3) where salary > 50000, + exp_lt = max(salary + 4) - min(salary - 4) where salary < 50000 + +; + +max:integer |max_f:integer | min:integer | min_f:integer |exp_gt:integer | exp_lt:integer +75000 |49820 | 25323 | 50062 |24941 | 24502 +; + +statsWithSubstitutedExpressionOverFilters +required_capability: per_agg_filtering +from employees +| stats sum = sum(salary), s_l = sum(salary) where salary < 50000, s_u = sum(salary) where salary > 50000, + count = count(salary), c_l = count(salary) where salary < 50000, c_u = count(salary) where salary > 50000, + avg = round(avg(salary), 2), a_l = round(avg(salary), 2) where salary < 50000, a_u = round(avg(salary),2) where salary > 50000 +; + +sum:l |s_l:l | s_u:l | count:l |c_l:l |c_u:l |avg:double |a_l:double | a_u:double +4824855 |2220951 | 2603904 | 100 |58 |42 |48248.55 |38292.26 | 61997.71 +; + + +statsWithFilterAndGroupBy +required_capability: per_agg_filtering +from employees +| stats m = max(height), + m_f = max(height + 1) where gender == "M" OR is_rehired is null + BY gender, is_rehired +| sort gender, is_rehired +; + +m:d |m_f:d |gender:s|is_rehired:bool +2.1 |null |F |false +2.1 |null |F |true +1.85|2.85 |F |null +2.1 |3.1 |M |false +2.1 |3.1 |M |true +2.01|3.01 |M |null +2.06|null |null |false +1.97|null |null |true +1.99|2.99 |null |null +; + +statsWithFilterOnGroupBy +required_capability: per_agg_filtering +from employees +| stats m_f = max(height) where gender == "M" BY gender +| sort gender +; + +m_f:d |gender:s +null |F +2.1 |M +null |null +; + +statsWithGroupByLiteral +required_capability: per_agg_filtering +from employees +| stats m = max(languages) by salary = 2 +; + +m:i |salary:i +5 |2 +; + + +statsWithFilterOnSameColumn +required_capability: per_agg_filtering +from employees +| stats m = max(languages), m_f = max(languages) where salary > 50000 by salary = 2 +| sort salary +; + +m:i |m_f:i |salary:i +5 |null |2 +; + +# the query is reused below in a multi-stats +statsWithFilteringAndGrouping +required_capability: per_agg_filtering +from employees +| stats c = count(), c_f = count(languages) where l > 1, + m_f = max(height) where salary > 50000 + by l = languages +| sort c +; + +c:l |c_f:l |m_f:d |l:i +10 |0 |2.08 |null +15 |0 |2.06 |1 +17 |17 |2.1 |3 +18 |18 |1.83 |4 +19 |19 |2.03 |2 +21 |21 |2.1 |5 +; + +multiStatsWithFiltering +required_capability: per_agg_filtering +from employees +| stats c = count(), c_f = count(languages) where l > 1, + m_f = max(height) where salary > 50000 + by l = languages +| stats c2 = count(), c2_f = count() where m_f > 2.06 , m2 = max(l), m2_f = max(l) where l > 1 by c +| sort c +; + +c2:l |c2_f:l |m2:i |m2_f:i |c:l +1 |1 |null |null |10 +1 |0 |1 |null |15 +1 |1 |3 |3 |17 +1 |0 |4 |4 |18 +1 |0 |2 |2 |19 +1 |1 |5 |5 |21 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec index 3d4d890546050..e7bf953f5e08d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec @@ -135,7 +135,7 @@ null |Swan |-8.46 |-8.46 Sanjiv |Zschoche |[-7.67, -3.25] |[-3.25, -7.67] |[-3, -8] |10053 ; -sortingOnSwappedFields +sortingOnSwappedFields#[skip:-8.13.3, reason:fixed in 8.13] FROM employees | EVAL name = last_name, last_name = first_name, first_name = name | WHERE first_name > "B" AND last_name IS NOT NULL @@ -157,7 +157,7 @@ Brattka | Charlene | Brattka Bridgland | Patricio | Bridgland ; -sortingOnSwappedFieldsNoKeep +sortingOnSwappedFieldsNoKeep#[skip:-8.13.3, reason:fixed in 8.13] // Note that this test requires all fields to be returned in order to test a specific code path in physical planning FROM employees | EVAL name = first_name, first_name = last_name, last_name = name diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json index dd4073d5dc7cf..39b1b10edd916 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json @@ -27,6 +27,10 @@ "message_in": { "type": "float", "time_series_metric": "counter" + }, + "message_out": { + "type": "integer", + "time_series_metric": "counter" } } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java similarity index 92% rename from x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java rename to x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java index e6f11ca1f44d2..e7da83a40fb20 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java @@ -14,9 +14,6 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; -import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; -import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; @@ -29,19 +26,13 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; -public class QueryStringFunctionIT extends AbstractEsqlIntegTestCase { +public class QueryStringIT extends AbstractEsqlIntegTestCase { @Before public void setupIndex() { createAndPopulateIndex(); } - @Override - protected EsqlQueryResponse run(EsqlQueryRequest request) { - assumeTrue("qstr function available in snapshot builds only", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); - return super.run(request); - } - public void testSimpleQueryString() { var query = """ FROM test diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java index 93701552b94aa..94fc4030f73a9 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java @@ -31,4 +31,10 @@ protected Geometry getQueryGeometry() { protected String castingFunction() { return "TO_CARTESIANSHAPE"; } + + @Override + protected double searchDistance() { + // We search much larger distances for Cartesian, to ensure we actually get results from the much wider data range + return 1e12; + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java index 871fb222de3d4..9bc3312fff63e 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java @@ -36,4 +36,9 @@ protected Geometry getQueryGeometry() { protected String castingFunction() { return "TO_GEOSHAPE"; } + + @Override + protected double searchDistance() { + return 10000000; + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java index 411106f008986..0acbe98022f02 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownPointsTestCase.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.spatial; +import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.GeometryValidator; import org.elasticsearch.geometry.utils.WellKnownText; @@ -20,6 +21,7 @@ import java.io.IOException; import java.text.ParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Locale; import static org.hamcrest.Matchers.closeTo; @@ -108,6 +110,61 @@ protected void assertFunction(String spatialFunction, String wkt, long expected, } } + public void testPushedDownDistanceSingleValue() throws RuntimeException { + assertPushedDownDistance(false); + } + + public void testPushedDownDistanceMultiValue() throws RuntimeException { + assertPushedDownDistance(true); + } + + private void assertPushedDownDistance(boolean multiValue) throws RuntimeException { + initIndexes(); + for (int i = 0; i < random().nextInt(50, 100); i++) { + if (multiValue) { + final String[] values = new String[randomIntBetween(1, 5)]; + for (int j = 0; j < values.length; j++) { + values[j] = "\"" + WellKnownText.toWKT(getIndexGeometry()) + "\""; + } + index("indexed", i + "", "{\"location\" : " + Arrays.toString(values) + " }"); + index("not-indexed", i + "", "{\"location\" : " + Arrays.toString(values) + " }"); + } else { + final String value = WellKnownText.toWKT(getIndexGeometry()); + index("indexed", i + "", "{\"location\" : \"" + value + "\" }"); + index("not-indexed", i + "", "{\"location\" : \"" + value + "\" }"); + } + } + + refresh("indexed", "not-indexed"); + + for (int i = 0; i < 10; i++) { + final Geometry geometry = getIndexGeometry(); + final String wkt = WellKnownText.toWKT(geometry); + assertDistanceFunction(wkt); + } + } + + protected abstract double searchDistance(); + + protected void assertDistanceFunction(String wkt) { + String spatialFunction = "ST_DISTANCE"; + String castingFunction = castingFunction().replaceAll("SHAPE", "POINT"); + final String query1 = String.format(Locale.ROOT, """ + FROM indexed | WHERE %s(location, %s("%s")) < %.1f | STATS COUNT(*) + """, spatialFunction, castingFunction, wkt, searchDistance()); + final String query2 = String.format(Locale.ROOT, """ + FROM not-indexed | WHERE %s(location, %s("%s")) < %.1f | STATS COUNT(*) + """, spatialFunction, castingFunction, wkt, searchDistance()); + try ( + EsqlQueryResponse response1 = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query1).get(); + EsqlQueryResponse response2 = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query2).get(); + ) { + Object indexedResult = response1.response().column(0).iterator().next(); + Object notIndexedResult = response2.response().column(0).iterator().next(); + assertEquals(spatialFunction, indexedResult, notIndexedResult); + } + } + private String toString(CentroidCalculator centroid) { return "Centroid (x:" + centroid.getX() + ", y:" + centroid.getY() + ")"; } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java index 9dff647763b6b..e7e0c785f50e5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java @@ -28,7 +28,7 @@ /** * Base class to check that a query than can be pushed down gives the same result * if it is actually pushed down and when it is executed by the compute engine, - * + *

* For doing that we create two indices, one fully indexed and another with index * and doc values disabled. Then we index the same data in both indices and we check * that the same ES|QL queries produce the same results in both. diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 0d8d3abf77ecc..b13606befd2a4 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -86,7 +86,6 @@ WHERE : 'where' -> pushMode(EXPRESSION_MODE); // MYCOMMAND : 'mycommand' -> ... DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats' -> pushMode(EXPRESSION_MODE); DEV_LOOKUP : {this.isDevVersion()}? 'lookup' -> pushMode(LOOKUP_MODE); -DEV_MATCH : {this.isDevVersion()}? 'match' -> pushMode(EXPRESSION_MODE); DEV_METRICS : {this.isDevVersion()}? 'metrics' -> pushMode(METRICS_MODE); // @@ -209,8 +208,8 @@ ASTERISK : '*'; SLASH : '/'; PERCENT : '%'; -// move it in the main section if the feature gets promoted -DEV_MATCH_OP : {this.isDevVersion()}? DEV_MATCH -> type(DEV_MATCH); +MATCH : 'match'; +NESTED_WHERE : {this.isDevVersion()}? WHERE -> type(WHERE); NAMED_OR_POSITIONAL_PARAM : PARAM (LETTER | UNDERSCORE) UNQUOTED_ID_BODY* diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 4fd37ab9900f2..4d1f426289149 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -16,51 +16,51 @@ STATS=15 WHERE=16 DEV_INLINESTATS=17 DEV_LOOKUP=18 -DEV_MATCH=19 -DEV_METRICS=20 -UNKNOWN_CMD=21 -LINE_COMMENT=22 -MULTILINE_COMMENT=23 -WS=24 -PIPE=25 -QUOTED_STRING=26 -INTEGER_LITERAL=27 -DECIMAL_LITERAL=28 -BY=29 -AND=30 -ASC=31 -ASSIGN=32 -CAST_OP=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -IN=39 -IS=40 -LAST=41 -LIKE=42 -LP=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -EQ=52 -CIEQ=53 -NEQ=54 -LT=55 -LTE=56 -GT=57 -GTE=58 -PLUS=59 -MINUS=60 -ASTERISK=61 -SLASH=62 -PERCENT=63 +DEV_METRICS=19 +UNKNOWN_CMD=20 +LINE_COMMENT=21 +MULTILINE_COMMENT=22 +WS=23 +PIPE=24 +QUOTED_STRING=25 +INTEGER_LITERAL=26 +DECIMAL_LITERAL=27 +BY=28 +AND=29 +ASC=30 +ASSIGN=31 +CAST_OP=32 +COMMA=33 +DESC=34 +DOT=35 +FALSE=36 +FIRST=37 +IN=38 +IS=39 +LAST=40 +LIKE=41 +LP=42 +NOT=43 +NULL=44 +NULLS=45 +OR=46 +PARAM=47 +RLIKE=48 +RP=49 +TRUE=50 +EQ=51 +CIEQ=52 +NEQ=53 +LT=54 +LTE=55 +GT=56 +GTE=57 +PLUS=58 +MINUS=59 +ASTERISK=60 +SLASH=61 +PERCENT=62 +MATCH=63 NAMED_OR_POSITIONAL_PARAM=64 OPENING_BRACKET=65 CLOSING_BRACKET=66 @@ -134,42 +134,43 @@ CLOSING_METRICS_WS=120 'sort'=14 'stats'=15 'where'=16 -'|'=25 -'by'=29 -'and'=30 -'asc'=31 -'='=32 -'::'=33 -','=34 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'in'=39 -'is'=40 -'last'=41 -'like'=42 -'('=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'=='=52 -'=~'=53 -'!='=54 -'<'=55 -'<='=56 -'>'=57 -'>='=58 -'+'=59 -'-'=60 -'*'=61 -'/'=62 -'%'=63 +'|'=24 +'by'=28 +'and'=29 +'asc'=30 +'='=31 +'::'=32 +','=33 +'desc'=34 +'.'=35 +'false'=36 +'first'=37 +'in'=38 +'is'=39 +'last'=40 +'like'=41 +'('=42 +'not'=43 +'null'=44 +'nulls'=45 +'or'=46 +'?'=47 +'rlike'=48 +')'=49 +'true'=50 +'=='=51 +'=~'=52 +'!='=53 +'<'=54 +'<='=55 +'>'=56 +'>='=57 +'+'=58 +'-'=59 +'*'=60 +'/'=61 +'%'=62 +'match'=63 ']'=66 'metadata'=75 'as'=84 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index b720ba98babf0..9a95e0e6726ba 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -77,7 +77,7 @@ regexBooleanExpression ; matchBooleanExpression - : valueExpression DEV_MATCH queryString=string + : valueExpression MATCH queryString=string ; valueExpression @@ -101,7 +101,13 @@ primaryExpression ; functionExpression - : identifierOrParameter LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP + : functionName LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP + ; + +functionName + // Additional function identifiers that are already a reserved word in the language + : MATCH + | identifierOrParameter ; dataType @@ -117,8 +123,7 @@ fields ; field - : booleanExpression - | qualifiedName ASSIGN booleanExpression + : (qualifiedName ASSIGN)? booleanExpression ; fromCommand @@ -126,8 +131,7 @@ fromCommand ; indexPattern - : clusterString COLON indexString - | indexString + : (clusterString COLON)? indexString ; clusterString @@ -153,7 +157,7 @@ deprecated_metadata ; metricsCommand - : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=fields? (BY grouping=fields)? + : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=aggFields? (BY grouping=fields)? ; evalCommand @@ -161,7 +165,15 @@ evalCommand ; statsCommand - : STATS stats=fields? (BY grouping=fields)? + : STATS stats=aggFields? (BY grouping=fields)? + ; + +aggFields + : aggField (COMMA aggField)* + ; + +aggField + : field {this.isDevVersion()}? (WHERE booleanExpression)? ; qualifiedName @@ -310,5 +322,5 @@ lookupCommand ; inlinestatsCommand - : DEV_INLINESTATS stats=fields (BY grouping=fields)? + : DEV_INLINESTATS stats=aggFields (BY grouping=fields)? ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens index 4fd37ab9900f2..4d1f426289149 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens @@ -16,51 +16,51 @@ STATS=15 WHERE=16 DEV_INLINESTATS=17 DEV_LOOKUP=18 -DEV_MATCH=19 -DEV_METRICS=20 -UNKNOWN_CMD=21 -LINE_COMMENT=22 -MULTILINE_COMMENT=23 -WS=24 -PIPE=25 -QUOTED_STRING=26 -INTEGER_LITERAL=27 -DECIMAL_LITERAL=28 -BY=29 -AND=30 -ASC=31 -ASSIGN=32 -CAST_OP=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -IN=39 -IS=40 -LAST=41 -LIKE=42 -LP=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -EQ=52 -CIEQ=53 -NEQ=54 -LT=55 -LTE=56 -GT=57 -GTE=58 -PLUS=59 -MINUS=60 -ASTERISK=61 -SLASH=62 -PERCENT=63 +DEV_METRICS=19 +UNKNOWN_CMD=20 +LINE_COMMENT=21 +MULTILINE_COMMENT=22 +WS=23 +PIPE=24 +QUOTED_STRING=25 +INTEGER_LITERAL=26 +DECIMAL_LITERAL=27 +BY=28 +AND=29 +ASC=30 +ASSIGN=31 +CAST_OP=32 +COMMA=33 +DESC=34 +DOT=35 +FALSE=36 +FIRST=37 +IN=38 +IS=39 +LAST=40 +LIKE=41 +LP=42 +NOT=43 +NULL=44 +NULLS=45 +OR=46 +PARAM=47 +RLIKE=48 +RP=49 +TRUE=50 +EQ=51 +CIEQ=52 +NEQ=53 +LT=54 +LTE=55 +GT=56 +GTE=57 +PLUS=58 +MINUS=59 +ASTERISK=60 +SLASH=61 +PERCENT=62 +MATCH=63 NAMED_OR_POSITIONAL_PARAM=64 OPENING_BRACKET=65 CLOSING_BRACKET=66 @@ -134,42 +134,43 @@ CLOSING_METRICS_WS=120 'sort'=14 'stats'=15 'where'=16 -'|'=25 -'by'=29 -'and'=30 -'asc'=31 -'='=32 -'::'=33 -','=34 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'in'=39 -'is'=40 -'last'=41 -'like'=42 -'('=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'=='=52 -'=~'=53 -'!='=54 -'<'=55 -'<='=56 -'>'=57 -'>='=58 -'+'=59 -'-'=60 -'*'=61 -'/'=62 -'%'=63 +'|'=24 +'by'=28 +'and'=29 +'asc'=30 +'='=31 +'::'=32 +','=33 +'desc'=34 +'.'=35 +'false'=36 +'first'=37 +'in'=38 +'is'=39 +'last'=40 +'like'=41 +'('=42 +'not'=43 +'null'=44 +'nulls'=45 +'or'=46 +'?'=47 +'rlike'=48 +')'=49 +'true'=50 +'=='=51 +'=~'=52 +'!='=53 +'<'=54 +'<='=55 +'>'=56 +'>='=57 +'+'=58 +'-'=59 +'*'=60 +'/'=61 +'%'=62 +'match'=63 ']'=66 'metadata'=75 'as'=84 diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java index f5684bcb4be18..22094f7e623e6 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java @@ -13,16 +13,16 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.expression.function.Warnings; /** * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Hypot}. * This class is generated. Do not edit it. */ public final class HypotEvaluator implements EvalOperator.ExpressionEvaluator { - private final Warnings warnings; + private final Source source; private final EvalOperator.ExpressionEvaluator n1; @@ -30,12 +30,14 @@ public final class HypotEvaluator implements EvalOperator.ExpressionEvaluator { private final DriverContext driverContext; + private Warnings warnings; + public HypotEvaluator(Source source, EvalOperator.ExpressionEvaluator n1, EvalOperator.ExpressionEvaluator n2, DriverContext driverContext) { + this.source = source; this.n1 = n1; this.n2 = n2; this.driverContext = driverContext; - this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); } @Override @@ -64,7 +66,7 @@ public DoubleBlock eval(int positionCount, DoubleBlock n1Block, DoubleBlock n2Bl } if (n1Block.getValueCount(p) != 1) { if (n1Block.getValueCount(p) > 1) { - warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); } result.appendNull(); continue position; @@ -75,7 +77,7 @@ public DoubleBlock eval(int positionCount, DoubleBlock n1Block, DoubleBlock n2Bl } if (n2Block.getValueCount(p) != 1) { if (n2Block.getValueCount(p) > 1) { - warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); } result.appendNull(); continue position; @@ -105,6 +107,18 @@ public void close() { Releasables.closeExpectNoException(n1, n2); } + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 18ee6b9417e5c..f5baaef4f579d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -344,7 +344,12 @@ public enum Cap { /** * QSTR function */ - QSTR_FUNCTION(true), + QSTR_FUNCTION, + + /** + * MATCH function + */ + MATCH_FUNCTION, /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. @@ -360,7 +365,17 @@ public enum Cap { /** * Support named parameters for field names. */ - NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES; + NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES, + + /** + * Fix sorting not allowed on _source and counters. + */ + SORTING_ON_SOURCE_AND_COUNTERS_FORBIDDEN, + + /** + * Allow filter per individual aggregation. + */ + PER_AGG_FILTERING; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 90957f55141b9..fe7b945a9b3c1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -488,6 +488,7 @@ private LogicalPlan resolveStats(Stats stats, List childrenOutput) { newAggregates.add(agg); } + // TODO: remove this when Stats interface is removed stats = changed.get() ? stats.with(stats.child(), groupings, newAggregates) : stats; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 647a29b71c5e1..ef39220d7ffcc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -19,15 +19,22 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -55,6 +62,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Stream; @@ -183,7 +191,7 @@ else if (p instanceof Lookup lookup) { checkOperationsOnUnsignedLong(p, failures); checkBinaryComparison(p, failures); - checkForSortOnSpatialTypes(p, failures); + checkForSortableDataTypes(p, failures); checkFilterMatchConditions(p, failures); checkFullTextQueryFunctions(p, failures); @@ -301,6 +309,29 @@ private static void checkInvalidNamedExpressionUsage( Set failures, int level ) { + // unwrap filtered expression + if (e instanceof FilteredExpression fe) { + e = fe.delegate(); + // make sure they work on aggregate functions + if (e.anyMatch(AggregateFunction.class::isInstance) == false) { + Expression filter = fe.filter(); + failures.add(fail(filter, "WHERE clause allowed only for aggregate functions, none found in [{}]", fe.sourceText())); + } + // but that the filter doesn't use grouping or aggregate functions + fe.filter().forEachDown(c -> { + if (c instanceof AggregateFunction af) { + failures.add( + fail(af, "cannot use aggregate function [{}] in aggregate WHERE clause [{}]", af.sourceText(), fe.sourceText()) + ); + } + // check the bucketing function against the group + else if (c instanceof GroupingFunction gf) { + if (Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) { + failures.add(fail(gf, "can only use grouping function [{}] part of the BY clause", gf.sourceText())); + } + } + }); + } // found an aggregate, constant or a group, bail out if (e instanceof AggregateFunction af) { af.field().forEachDown(AggregateFunction.class, f -> { @@ -312,7 +343,7 @@ private static void checkInvalidNamedExpressionUsage( } else if (e instanceof GroupingFunction gf) { // optimizer will later unroll expressions with aggs and non-aggs with a grouping function into an EVAL, but that will no longer // be verified (by check above in checkAggregate()), so do it explicitly here - if (groups.stream().anyMatch(ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) { + if (Expressions.anyMatch(groups, ex -> ex instanceof Alias a && a.child().semanticEquals(gf)) == false) { failures.add(fail(gf, "can only use grouping function [{}] part of the BY clause", gf.sourceText())); } else if (level == 0) { addFailureOnGroupingUsedNakedInAggs(failures, gf, "function"); @@ -548,12 +579,12 @@ private static Failure validateUnsignedLongNegation(Neg neg) { } /** - * Makes sure that spatial types do not appear in sorting contexts. + * Some datatypes are not sortable */ - private static void checkForSortOnSpatialTypes(LogicalPlan p, Set localFailures) { + private static void checkForSortableDataTypes(LogicalPlan p, Set localFailures) { if (p instanceof OrderBy ob) { ob.order().forEach(order -> { - if (DataType.isSpatial(order.dataType())) { + if (DataType.isSortable(order.dataType()) == false) { localFailures.add(fail(order, "cannot sort on " + order.dataType().typeName())); } }); @@ -644,27 +675,105 @@ private static void checkFilterMatchConditions(LogicalPlan plan, Set fa private static void checkFullTextQueryFunctions(LogicalPlan plan, Set failures) { if (plan instanceof Filter f) { Expression condition = f.condition(); - if (condition instanceof FullTextFunction ftf) { - // Similar to cases present in org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters - - // we can't check if it can be pushed down as we don't have yet information about the fields present in the - // StringQueryPredicate - plan.forEachDown(LogicalPlan.class, lp -> { - if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) { - failures.add( - fail( - plan, - "[{}] function cannot be used after {}", - ftf.functionName(), - lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) - ) - ); - } - }); - } + checkCommandsBeforeQueryStringFunction(plan, condition, failures); + checkCommandsBeforeMatchFunction(plan, condition, failures); + checkFullTextFunctionsConditions(condition, failures); + checkFullTextFunctionsParents(condition, failures); } else { plan.forEachExpression(FullTextFunction.class, ftf -> { failures.add(fail(ftf, "[{}] function is only supported in WHERE commands", ftf.functionName())); }); } } + + private static void checkCommandsBeforeQueryStringFunction(LogicalPlan plan, Expression condition, Set failures) { + condition.forEachDown(QueryString.class, qsf -> { + plan.forEachDown(LogicalPlan.class, lp -> { + if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) { + failures.add( + fail( + plan, + "[{}] function cannot be used after {}", + qsf.functionName(), + lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) + ) + ); + } + }); + }); + } + + private static void checkCommandsBeforeMatchFunction(LogicalPlan plan, Expression condition, Set failures) { + condition.forEachDown(Match.class, qsf -> { + plan.forEachDown(LogicalPlan.class, lp -> { + if (lp instanceof Limit) { + failures.add( + fail( + plan, + "[{}] function cannot be used after {}", + qsf.functionName(), + lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) + ) + ); + } + }); + }); + } + + private static void checkFullTextFunctionsConditions(Expression condition, Set failures) { + condition.forEachUp(Or.class, or -> { + checkFullTextFunctionInDisjunction(failures, or, or.left()); + checkFullTextFunctionInDisjunction(failures, or, or.right()); + }); + } + + private static void checkFullTextFunctionInDisjunction(Set failures, Or or, Expression left) { + left.forEachDown(FullTextFunction.class, ftf -> { + failures.add( + fail( + or, + "Invalid condition [{}]. Function {} can't be used as part of an or condition", + or.sourceText(), + ftf.functionName() + ) + ); + }); + } + + private static void checkFullTextFunctionsParents(Expression condition, Set failures) { + forEachFullTextFunctionParent(condition, (ftf, parent) -> { + if ((parent instanceof FullTextFunction == false) + && (parent instanceof BinaryLogic == false) + && (parent instanceof Not == false)) { + failures.add( + fail( + condition, + "Invalid condition [{}]. Function {} can't be used with {}", + condition.sourceText(), + ftf.functionName(), + ((Function) parent).functionName() + ) + ); + } + }); + } + + /** + * Executes the action on every parent of a FullTextFunction in the condition if it is found + * + * @param action the action to execute for each parent of a FullTextFunction + */ + private static FullTextFunction forEachFullTextFunctionParent(Expression condition, BiConsumer action) { + if (condition instanceof FullTextFunction ftf) { + return ftf; + } + for (Expression child : condition.children()) { + FullTextFunction foundMatchingChild = forEachFullTextFunctionParent(child, action); + if (foundMatchingChild != null) { + action.accept(foundMatchingChild, condition); + return foundMatchingChild; + } + } + return null; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 3b1225555b297..66151275fc2e8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -32,7 +32,8 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg; -import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryStringFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -258,19 +259,21 @@ private FunctionDefinition[][] functions() { // grouping functions new FunctionDefinition[] { def(Bucket.class, Bucket::new, "bucket", "bin"), }, // aggregate functions + // since they declare two public constructors - one with filter (for nested where) and one without + // use casting to disambiguate between the two new FunctionDefinition[] { - def(Avg.class, Avg::new, "avg"), - def(Count.class, Count::new, "count"), - def(CountDistinct.class, CountDistinct::new, "count_distinct"), - def(Max.class, Max::new, "max"), - def(Median.class, Median::new, "median"), - def(MedianAbsoluteDeviation.class, MedianAbsoluteDeviation::new, "median_absolute_deviation"), - def(Min.class, Min::new, "min"), - def(Percentile.class, Percentile::new, "percentile"), - def(Sum.class, Sum::new, "sum"), - def(Top.class, Top::new, "top"), - def(Values.class, Values::new, "values"), - def(WeightedAvg.class, WeightedAvg::new, "weighted_avg") }, + def(Avg.class, uni(Avg::new), "avg"), + def(Count.class, uni(Count::new), "count"), + def(CountDistinct.class, bi(CountDistinct::new), "count_distinct"), + def(Max.class, uni(Max::new), "max"), + def(Median.class, uni(Median::new), "median"), + def(MedianAbsoluteDeviation.class, uni(MedianAbsoluteDeviation::new), "median_absolute_deviation"), + def(Min.class, uni(Min::new), "min"), + def(Percentile.class, bi(Percentile::new), "percentile"), + def(Sum.class, uni(Sum::new), "sum"), + def(Top.class, tri(Top::new), "top"), + def(Values.class, uni(Values::new), "values"), + def(WeightedAvg.class, bi(WeightedAvg::new), "weighted_avg") }, // math new FunctionDefinition[] { def(Abs.class, Abs::new, "abs"), @@ -385,7 +388,9 @@ private FunctionDefinition[][] functions() { def(MvSlice.class, MvSlice::new, "mv_slice"), def(MvZip.class, MvZip::new, "mv_zip"), def(MvSum.class, MvSum::new, "mv_sum"), - def(Split.class, Split::new, "split") } }; + def(Split.class, Split::new, "split") }, + // fulltext functions + new FunctionDefinition[] { def(Match.class, Match::new, "match"), def(QueryString.class, QueryString::new, "qstr") } }; } @@ -393,9 +398,7 @@ private static FunctionDefinition[][] snapshotFunctions() { return new FunctionDefinition[][] { new FunctionDefinition[] { def(Categorize.class, Categorize::new, "categorize"), - def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), - // Full text functions - def(QueryStringFunction.class, QueryStringFunction::new, "qstr") } }; + def(Rate.class, Rate::withUnresolvedTimestamp, "rate") } }; } public EsqlFunctionRegistry snapshotRegistry() { @@ -481,11 +484,10 @@ public static DataType getTargetType(String[] names) { } public static FunctionDescription description(FunctionDefinition def) { - var constructors = def.clazz().getConstructors(); - if (constructors.length == 0) { + Constructor constructor = constructorFor(def.clazz()); + if (constructor == null) { return new FunctionDescription(def.name(), List.of(), null, null, false, false); } - Constructor constructor = constructors[0]; FunctionInfo functionInfo = functionInfo(def); String functionDescription = functionInfo == null ? "" : functionInfo.description().replace('\n', ' '); String[] returnType = functionInfo == null ? new String[] { "?" } : removeUnderConstruction(functionInfo.returnType()); @@ -522,14 +524,29 @@ private static String[] removeUnderConstruction(String[] types) { } public static FunctionInfo functionInfo(FunctionDefinition def) { - var constructors = def.clazz().getConstructors(); - if (constructors.length == 0) { + Constructor constructor = constructorFor(def.clazz()); + if (constructor == null) { return null; } - Constructor constructor = constructors[0]; return constructor.getAnnotation(FunctionInfo.class); } + private static Constructor constructorFor(Class clazz) { + Constructor[] constructors = clazz.getConstructors(); + if (constructors.length == 0) { + return null; + } + // when dealing with multiple, pick the constructor exposing the FunctionInfo annotation + if (constructors.length > 1) { + for (Constructor constructor : constructors) { + if (constructor.getAnnotation(FunctionInfo.class) != null) { + return constructor; + } + } + } + return constructors[0]; + } + private void buildDataTypesForStringLiteralConversion(FunctionDefinition[]... groupFunctions) { for (FunctionDefinition[] group : groupFunctions) { for (FunctionDefinition def : group) { @@ -912,15 +929,19 @@ protected interface TernaryConfigurationAwareBuilder { } // - // Utility method for extra argument extraction. + // Utility functions to help disambiguate the method handle passed in. + // They work by providing additional method information to help the compiler know which method to pick. // - protected static Boolean asBool(Object[] extras) { - if (CollectionUtils.isEmpty(extras)) { - return null; - } - if (extras.length != 1 || (extras[0] instanceof Boolean) == false) { - throw new QlIllegalArgumentException("Invalid number and types of arguments given to function definition"); - } - return (Boolean) extras[0]; + private static BiFunction uni(BiFunction function) { + return function; } + + private static BinaryBuilder bi(BinaryBuilder function) { + return function; + } + + private static TernaryBuilder tri(TernaryBuilder function) { + return function; + } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java index f0acac0e9744e..f7a74cc2ae93f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java @@ -6,10 +6,12 @@ */ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -20,8 +22,8 @@ import java.util.List; import java.util.Objects; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; /** @@ -52,25 +54,51 @@ public static List getNamedWriteables() { private final Expression field; private final List parameters; + private final Expression filter; protected AggregateFunction(Source source, Expression field) { - this(source, field, emptyList()); + this(source, field, Literal.TRUE, emptyList()); } protected AggregateFunction(Source source, Expression field, List parameters) { - super(source, CollectionUtils.combine(singletonList(field), parameters)); + this(source, field, Literal.TRUE, parameters); + } + + protected AggregateFunction(Source source, Expression field, Expression filter, List parameters) { + super(source, CollectionUtils.combine(asList(field, filter), parameters)); this.field = field; + this.filter = filter; this.parameters = parameters; } protected AggregateFunction(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class) + : emptyList() + ); } @Override - public void writeTo(StreamOutput out) throws IOException { + public final void writeTo(StreamOutput out) throws IOException { Source.EMPTY.writeTo(out); out.writeNamedWriteable(field); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)) { + out.writeNamedWriteable(filter); + out.writeNamedWriteableCollection(parameters); + } else { + deprecatedWriteParams(out); + } + } + + @Deprecated(since = "8.16", forRemoval = true) + protected void deprecatedWriteParams(StreamOutput out) throws IOException { + // } public Expression field() { @@ -81,12 +109,12 @@ public List parameters() { return parameters; } - /** - * Returns the input expressions used in aggregation. - * Defaults to a list containing the only the input field. - */ - public List inputExpressions() { - return List.of(field); + public boolean hasFilter() { + return filter != null && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold()) == false); + } + + public Expression filter() { + return filter; } @Override @@ -94,6 +122,18 @@ protected TypeResolution resolveType() { return TypeResolutions.isExact(field, sourceText(), DEFAULT); } + /** + * Attach a filter to the aggregate function. + */ + public abstract AggregateFunction withFilter(Expression filter); + + public AggregateFunction withParameters(List parameters) { + if (parameters == this.parameters) { + return this; + } + return (AggregateFunction) replaceChildren(CollectionUtils.combine(asList(field, filter), parameters)); + } + @Override public int hashCode() { // NB: the hashcode is currently used for key generation so @@ -105,7 +145,9 @@ public int hashCode() { public boolean equals(Object obj) { if (super.equals(obj)) { AggregateFunction other = (AggregateFunction) obj; - return Objects.equals(other.field(), field()) && Objects.equals(other.parameters(), parameters()); + return Objects.equals(other.field(), field()) + && Objects.equals(other.filter(), filter()) + && Objects.equals(other.parameters(), parameters()); } return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java index b5c0b8e5ffdc8..82c0f9d24899e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -23,6 +24,7 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -44,7 +46,11 @@ public class Avg extends AggregateFunction implements SurrogateExpression { ) } ) public Avg(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { - super(source, field); + this(source, field, Literal.TRUE); + } + + public Avg(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } @Override @@ -74,12 +80,17 @@ public DataType dataType() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Avg::new, field()); + return NodeInfo.create(this, Avg::new, field(), filter()); } @Override public Avg replaceChildren(List newChildren) { - return new Avg(source(), newChildren.get(0)); + return new Avg(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public Avg withFilter(Expression filter) { + return new Avg(source(), field(), filter); } @Override @@ -87,6 +98,8 @@ public Expression surrogate() { var s = source(); var field = field(); - return field().foldable() ? new MvAvg(s, field) : new Div(s, new Sum(s, field), new Count(s, field), dataType()); + return field().foldable() + ? new MvAvg(s, field) + : new Div(s, new Sum(s, field, filter()), new Count(s, field, filter()), dataType()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 9b6190408dbd4..fa8a9e7d8c837 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -30,10 +30,11 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -public class Count extends AggregateFunction implements EnclosedAgg, ToAggregator, SurrogateExpression { +public class Count extends AggregateFunction implements ToAggregator, SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Count", Count::new); @FunctionInfo( @@ -83,7 +84,11 @@ public Count( description = "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)." ) Expression field ) { - super(source, field); + this(source, field, Literal.TRUE); + } + + public Count(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } private Count(StreamInput in) throws IOException { @@ -97,17 +102,17 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Count::new, field()); + return NodeInfo.create(this, Count::new, field(), filter()); } @Override - public Count replaceChildren(List newChildren) { - return new Count(source(), newChildren.get(0)); + public AggregateFunction withFilter(Expression filter) { + return new Count(source(), field(), filter); } @Override - public String innerName() { - return "count"; + public Count replaceChildren(List newChildren) { + return new Count(source(), newChildren.get(0), newChildren.get(1)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 858c6e659449c..2550e5bdcf515 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -43,6 +44,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isWholeNumber; +import static org.elasticsearch.xpack.esql.core.util.CollectionUtils.nullSafeList; public class CountDistinct extends AggregateFunction implements OptionalArgument, ToAggregator, SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -112,22 +114,33 @@ public CountDistinct( + "same effect as a threshold of 40000. The default value is 3000." ) Expression precision ) { - super(source, field, precision != null ? List.of(precision) : List.of()); - this.precision = precision; + this(source, field, Literal.TRUE, precision); + } + + public CountDistinct(Source source, Expression field, Expression filter, Expression precision) { + this(source, field, filter, precision != null ? List.of(precision) : List.of()); + } + + private CountDistinct(Source source, Expression field, Expression filter, List params) { + super(source, field, filter, params); + this.precision = params.size() > 0 ? params.get(0) : null; } private CountDistinct(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.readOptionalNamedWriteable(Expression.class) + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class) + : nullSafeList(in.readOptionalNamedWriteable(Expression.class)) ); } @Override - public void writeTo(StreamOutput out) throws IOException { - Source.EMPTY.writeTo(out); - out.writeNamedWriteable(field()); + protected void deprecatedWriteParams(StreamOutput out) throws IOException { out.writeOptionalNamedWriteable(precision); } @@ -138,12 +151,17 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, CountDistinct::new, field(), precision); + return NodeInfo.create(this, CountDistinct::new, field(), filter(), precision); } @Override public CountDistinct replaceChildren(List newChildren) { - return new CountDistinct(source(), newChildren.get(0), newChildren.size() > 1 ? newChildren.get(1) : null); + return new CountDistinct(source(), newChildren.get(0), newChildren.get(1), newChildren.size() > 2 ? newChildren.get(2) : null); + } + + @Override + public CountDistinct withFilter(Expression filter) { + return new CountDistinct(source(), field(), filter, precision); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/EnclosedAgg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/EnclosedAgg.java deleted file mode 100644 index 951a991da376b..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/EnclosedAgg.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.expression.function.aggregate; - -// Agg 'enclosed' by another agg. Used for agg that return multiple embedded aggs (like MatrixStats) -public interface EnclosedAgg { - - String innerName(); -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FilteredExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FilteredExpression.java new file mode 100644 index 0000000000000..97c6fb6dbd887 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FilteredExpression.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; + +import static java.util.Arrays.asList; + +/** + * Basic wrapper for expressions declared with a nested filter (typically in stats). + * Used during parsing to attach the filter to the nested expression - it is expected the two + * get fused later on. + */ +// TODO: This class should implement SurrogateExpression but it doesn't due to its use on folding aggregates +// see https://github.com/elastic/elasticsearch/issues/100634#issuecomment-2400665066 +public class FilteredExpression extends Expression { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "FilteredExpression", + FilteredExpression::new + ); + + private final Expression delegate; + private final Expression filter; + + public FilteredExpression(Source source, Expression delegate, Expression filter) { + super(source, asList(delegate, filter)); + this.delegate = delegate; + this.filter = filter; + } + + public FilteredExpression(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + } + + public Expression surrogate() { + return delegate.transformUp(AggregateFunction.class, af -> af.withFilter(filter)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + out.writeNamedWriteable(delegate); + out.writeNamedWriteable(filter); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + public Expression delegate() { + return delegate; + } + + public Expression filter() { + return filter; + } + + @Override + public DataType dataType() { + return delegate.dataType(); + } + + @Override + public Nullability nullable() { + return delegate.nullable(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, FilteredExpression::new, delegate, filter); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new FilteredExpression(source(), newChildren.get(0), newChildren.get(1)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java index 593e6fa463371..0f9037a28d7d7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -21,6 +22,7 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -44,18 +46,29 @@ public class FromPartial extends AggregateFunction implements ToAggregator { private final Expression function; public FromPartial(Source source, Expression field, Expression function) { - super(source, field, List.of(function)); + this(source, field, Literal.TRUE, function); + } + + public FromPartial(Source source, Expression field, Expression filter, Expression function) { + super(source, field, filter, List.of(function)); this.function = function; } private FromPartial(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) + : in.readNamedWriteable(Expression.class) + ); } @Override - public void writeTo(StreamOutput out) throws IOException { - source().writeTo(out); - out.writeNamedWriteable(field()); + protected void deprecatedWriteParams(StreamOutput out) throws IOException { out.writeNamedWriteable(function); } @@ -85,12 +98,17 @@ public AttributeSet references() { @Override public Expression replaceChildren(List newChildren) { - return new FromPartial(source(), newChildren.get(0), newChildren.get(1)); + return new FromPartial(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); } @Override protected NodeInfo info() { - return NodeInfo.create(this, FromPartial::new, field(), function); + return NodeInfo.create(this, FromPartial::new, field(), filter(), function); + } + + @Override + public FromPartial withFilter(Expression filter) { + return new FromPartial(source(), field(), filter, function); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index e7f790f90803a..47d74c71d9cc5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -18,6 +18,7 @@ import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -32,6 +33,7 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; @@ -61,7 +63,11 @@ public Max( type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" } ) Expression field ) { - super(source, field); + this(source, field, Literal.TRUE); + } + + public Max(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } private Max(StreamInput in) throws IOException { @@ -73,14 +79,19 @@ public String getWriteableName() { return ENTRY.name; } + @Override + public Max withFilter(Expression filter) { + return new Max(source(), field(), filter); + } + @Override protected NodeInfo info() { - return NodeInfo.create(this, Max::new, field()); + return NodeInfo.create(this, Max::new, field(), filter()); } @Override public Max replaceChildren(List newChildren) { - return new Max(source(), newChildren.get(0)); + return new Max(source(), newChildren.get(0), newChildren.get(1)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java index 348fef577c934..c47fa612c1c49 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -55,7 +56,11 @@ public class Median extends AggregateFunction implements SurrogateExpression { ), } ) public Median(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { - super(source, field); + this(source, field, Literal.TRUE); + } + + public Median(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } @Override @@ -85,12 +90,17 @@ public DataType dataType() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Median::new, field()); + return NodeInfo.create(this, Median::new, field(), filter()); } @Override public Median replaceChildren(List newChildren) { - return new Median(source(), newChildren.get(0)); + return new Median(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public AggregateFunction withFilter(Expression filter) { + return new Median(source(), field(), filter); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java index 23a6b23a35cde..dfcbd6d22abae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.aggregation.MedianAbsoluteDeviationIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MedianAbsoluteDeviationLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.SurrogateExpression; @@ -26,6 +27,8 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; + public class MedianAbsoluteDeviation extends NumericAggregate implements SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, @@ -64,7 +67,11 @@ public class MedianAbsoluteDeviation extends NumericAggregate implements Surroga ), } ) public MedianAbsoluteDeviation(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { - super(source, field); + this(source, field, Literal.TRUE); + } + + public MedianAbsoluteDeviation(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } private MedianAbsoluteDeviation(StreamInput in) throws IOException { @@ -78,12 +85,17 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, MedianAbsoluteDeviation::new, field()); + return NodeInfo.create(this, MedianAbsoluteDeviation::new, field(), filter()); } @Override public MedianAbsoluteDeviation replaceChildren(List newChildren) { - return new MedianAbsoluteDeviation(source(), newChildren.get(0)); + return new MedianAbsoluteDeviation(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public MedianAbsoluteDeviation withFilter(Expression filter) { + return new MedianAbsoluteDeviation(source(), field(), filter); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index 6866811995059..ce69decca8e81 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -18,6 +18,7 @@ import org.elasticsearch.compute.aggregation.MinLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -32,6 +33,7 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; @@ -61,7 +63,11 @@ public Min( type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" } ) Expression field ) { - super(source, field); + this(source, field, Literal.TRUE); + } + + public Min(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } private Min(StreamInput in) throws IOException { @@ -75,12 +81,17 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Min::new, field()); + return NodeInfo.create(this, Min::new, field(), filter()); } @Override public Min replaceChildren(List newChildren) { - return new Min(source(), newChildren.get(0)); + return new Min(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public Min withFilter(Expression filter) { + return new Min(source(), field(), filter); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java index e7825a1d11704..5c639c465c649 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java @@ -49,6 +49,10 @@ public abstract class NumericAggregate extends AggregateFunction implements ToAg super(source, field, parameters); } + NumericAggregate(Source source, Expression field, Expression filter, List parameters) { + super(source, field, filter, parameters); + } + NumericAggregate(Source source, Expression field) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java index 0d5dd4b66501c..febd9f28b2291 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -15,6 +16,7 @@ import org.elasticsearch.compute.aggregation.PercentileIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.PercentileLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -29,6 +31,7 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable; @@ -77,19 +80,30 @@ public Percentile( @Param(name = "number", type = { "double", "integer", "long" }) Expression field, @Param(name = "percentile", type = { "double", "integer", "long" }) Expression percentile ) { - super(source, field, List.of(percentile)); + this(source, field, Literal.TRUE, percentile); + } + + public Percentile(Source source, Expression field, Expression filter, Expression percentile) { + super(source, field, filter, singletonList(percentile)); this.percentile = percentile; } private Percentile(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) + : in.readNamedWriteable(Expression.class) + ); } @Override - public void writeTo(StreamOutput out) throws IOException { - Source.EMPTY.writeTo(out); - out.writeNamedWriteable(children().get(0)); - out.writeNamedWriteable(children().get(1)); + protected void deprecatedWriteParams(StreamOutput out) throws IOException { + out.writeNamedWriteable(percentile); } @Override @@ -99,12 +113,17 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Percentile::new, field(), percentile); + return NodeInfo.create(this, Percentile::new, field(), filter(), percentile); } @Override public Percentile replaceChildren(List newChildren) { - return new Percentile(source(), newChildren.get(0), newChildren.get(1)); + return new Percentile(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } + + @Override + public Percentile withFilter(Expression filter) { + return new Percentile(source(), field(), filter, percentile); } public Expression percentile() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index f5597b7d64e81..b7b04658f8d58 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -17,6 +18,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -34,6 +36,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.util.CollectionUtils.nullSafeList; public class Rate extends AggregateFunction implements OptionalArgument, ToAggregator { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Rate", Rate::new); @@ -53,7 +56,16 @@ public Rate( Expression timestamp, @Param(optional = true, name = "unit", type = { "time_duration" }, description = "the unit") Expression unit ) { - super(source, field, unit != null ? List.of(timestamp, unit) : List.of(timestamp)); + this(source, field, Literal.TRUE, timestamp, unit); + } + + // compatibility constructor used when reading from the stream + private Rate(Source source, Expression field, Expression filter, List children) { + this(source, field, filter, children.get(0), children.size() > 1 ? children.get(1) : null); + } + + private Rate(Source source, Expression field, Expression filter, Expression timestamp, Expression unit) { + super(source, field, filter, unit != null ? List.of(timestamp, unit) : List.of(timestamp)); this.timestamp = timestamp; this.unit = unit; } @@ -62,15 +74,17 @@ public Rate(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.readNamedWriteable(Expression.class), - in.readOptionalNamedWriteable(Expression.class) + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class) + : nullSafeList(in.readNamedWriteable(Expression.class), in.readOptionalNamedWriteable(Expression.class)) ); } @Override - public void writeTo(StreamOutput out) throws IOException { - source().writeTo(out); - out.writeNamedWriteable(field()); + protected void deprecatedWriteParams(StreamOutput out) throws IOException { out.writeNamedWriteable(timestamp); out.writeOptionalNamedWriteable(unit); } @@ -92,20 +106,25 @@ protected NodeInfo info() { @Override public Rate replaceChildren(List newChildren) { if (unit != null) { - if (newChildren.size() == 3) { - return new Rate(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + if (newChildren.size() == 4) { + return new Rate(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3)); } - assert false : "expected 3 children for field, @timestamp, and unit; got " + newChildren; - throw new IllegalArgumentException("expected 3 children for field, @timestamp, and unit; got " + newChildren); + assert false : "expected 4 children for field, filter, @timestamp, and unit; got " + newChildren; + throw new IllegalArgumentException("expected 4 children for field, filter, @timestamp, and unit; got " + newChildren); } else { - if (newChildren.size() == 2) { - return new Rate(source(), newChildren.get(0), newChildren.get(1), null); + if (newChildren.size() == 3) { + return new Rate(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), null); } - assert false : "expected 2 children for field and @timestamp; got " + newChildren; - throw new IllegalArgumentException("expected 2 children for field and @timestamp; got " + newChildren); + assert false : "expected 3 children for field, filter and @timestamp; got " + newChildren; + throw new IllegalArgumentException("expected 3 children for field, filter and @timestamp; got " + newChildren); } } + @Override + public Rate withFilter(Expression filter) { + return new Rate(source(), field(), filter, timestamp, unit); + } + @Override public DataType dataType() { return DataType.DOUBLE; @@ -115,7 +134,7 @@ public DataType dataType() { protected TypeResolution resolveType() { TypeResolution resolution = isType( field(), - dt -> dt == DataType.COUNTER_LONG || dt == DataType.COUNTER_INTEGER || dt == DataType.COUNTER_DOUBLE, + dt -> DataType.isCounter(dt), sourceText(), FIRST, "counter_long", @@ -149,11 +168,6 @@ long unitInMillis() { throw new IllegalArgumentException("function [" + sourceText() + "] has invalid unit [" + unit.sourceText() + "]"); } - @Override - public List inputExpressions() { - return List.of(field(), timestamp); - } - @Override public AggregatorFunctionSupplier supplier(List inputChannels) { if (inputChannels.size() != 2 && inputChannels.size() != 3) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java index 5cb7edf2581d5..87eec540932b1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialAggregateFunction.java @@ -14,6 +14,8 @@ import java.io.IOException; import java.util.Objects; +import static java.util.Collections.emptyList; + /** * All spatial aggregate functions extend this class to enable the planning of reading from doc values for higher performance. * The AggregateMapper class will generate multiple aggregation functions for each combination, allowing the planner to @@ -22,8 +24,8 @@ public abstract class SpatialAggregateFunction extends AggregateFunction { protected final boolean useDocValues; - protected SpatialAggregateFunction(Source source, Expression field, boolean useDocValues) { - super(source, field); + protected SpatialAggregateFunction(Source source, Expression field, Expression filter, boolean useDocValues) { + super(source, field, filter, emptyList()); this.useDocValues = useDocValues; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java index b9cd99f8eb7f0..aad95c07e3492 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java @@ -15,6 +15,7 @@ import org.elasticsearch.compute.aggregation.spatial.SpatialCentroidGeoPointSourceValuesAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -46,11 +47,11 @@ public class SpatialCentroid extends SpatialAggregateFunction implements ToAggre examples = @Example(file = "spatial", tag = "st_centroid_agg-airports") ) public SpatialCentroid(Source source, @Param(name = "field", type = { "geo_point", "cartesian_point" }) Expression field) { - super(source, field, false); + this(source, field, Literal.TRUE, false); } - private SpatialCentroid(Source source, Expression field, boolean useDocValues) { - super(source, field, useDocValues); + private SpatialCentroid(Source source, Expression field, Expression filter, boolean useDocValues) { + super(source, field, filter, useDocValues); } private SpatialCentroid(StreamInput in) throws IOException { @@ -62,9 +63,14 @@ public String getWriteableName() { return ENTRY.name; } + @Override + public SpatialCentroid withFilter(Expression filter) { + return new SpatialCentroid(source(), field(), filter, useDocValues); + } + @Override public SpatialCentroid withDocValues() { - return new SpatialCentroid(source(), field(), true); + return new SpatialCentroid(source(), field(), filter(), true); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java index 4f85a15732a6f..37c2abaae1e4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; @@ -53,7 +54,11 @@ public class Sum extends NumericAggregate implements SurrogateExpression { ) } ) public Sum(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { - super(source, field); + this(source, field, Literal.TRUE); + } + + public Sum(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } private Sum(StreamInput in) throws IOException { @@ -67,12 +72,17 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Sum::new, field()); + return NodeInfo.create(this, Sum::new, field(), filter()); } @Override public Sum replaceChildren(List newChildren) { - return new Sum(source(), newChildren.get(0)); + return new Sum(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public Sum withFilter(Expression filter) { + return new Sum(source(), field(), filter); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java index c1da400185944..cffac616b3c8c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -21,6 +22,7 @@ import org.elasticsearch.compute.aggregation.ToPartialGroupingAggregatorFunction; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -66,18 +68,29 @@ public class ToPartial extends AggregateFunction implements ToAggregator { private final Expression function; public ToPartial(Source source, Expression field, Expression function) { - super(source, field, List.of(function)); + this(source, field, Literal.TRUE, function); + } + + public ToPartial(Source source, Expression field, Expression filter, Expression function) { + super(source, field, filter, List.of(function)); this.function = function; } private ToPartial(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) + : in.readNamedWriteable(Expression.class) + ); } @Override - public void writeTo(StreamOutput out) throws IOException { - source().writeTo(out); - out.writeNamedWriteable(field()); + protected void deprecatedWriteParams(StreamOutput out) throws IOException { out.writeNamedWriteable(function); } @@ -102,12 +115,17 @@ protected TypeResolution resolveType() { @Override public Expression replaceChildren(List newChildren) { - return new ToPartial(source(), newChildren.get(0), newChildren.get(1)); + return new ToPartial(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } + + @Override + public ToPartial withFilter(Expression filter) { + return new ToPartial(source(), field(), filter(), function); } @Override - protected NodeInfo info() { - return NodeInfo.create(this, ToPartial::new, field(), function); + protected NodeInfo info() { + return NodeInfo.create(this, ToPartial::new, field(), filter(), function); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java index cb1b0f0cad895..4f81e0a897f9c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -20,6 +21,7 @@ import org.elasticsearch.compute.aggregation.TopLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -31,9 +33,9 @@ import org.elasticsearch.xpack.esql.planner.ToAggregator; import java.io.IOException; -import java.util.Arrays; import java.util.List; +import static java.util.Arrays.asList; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -68,26 +70,37 @@ public Top( description = "The order to calculate the top values. Either `asc` or `desc`." ) Expression order ) { - super(source, field, Arrays.asList(limit, order)); + this(source, field, Literal.TRUE, limit, order); + } + + public Top(Source source, Expression field, Expression filter, Expression limit, Expression order) { + super(source, field, filter, asList(limit, order)); } private Top(StreamInput in) throws IOException { - this( + super( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.readNamedWriteable(Expression.class), - in.readNamedWriteable(Expression.class) + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class) + : asList(in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)) ); } @Override - public void writeTo(StreamOutput out) throws IOException { - source().writeTo(out); - List fields = children(); - assert fields.size() == 3; - out.writeNamedWriteable(fields.get(0)); - out.writeNamedWriteable(fields.get(1)); - out.writeNamedWriteable(fields.get(2)); + protected void deprecatedWriteParams(StreamOutput out) throws IOException { + List params = parameters(); + assert params.size() == 2; + out.writeNamedWriteable(params.get(0)); + out.writeNamedWriteable(params.get(1)); + } + + @Override + public Top withFilter(Expression filter) { + return new Top(source(), field(), filter, limitField(), orderField()); } @Override @@ -167,12 +180,12 @@ public DataType dataType() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Top::new, children().get(0), children().get(1), children().get(2)); + return NodeInfo.create(this, Top::new, field(), filter(), limitField(), orderField()); } @Override public Top replaceChildren(List newChildren) { - return new Top(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + return new Top(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java index 136e1233601f9..a844b981c95d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.aggregation.ValuesLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -29,6 +30,7 @@ import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; @@ -56,7 +58,11 @@ public Values( Source source, @Param(name = "field", type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }) Expression v ) { - super(source, v); + this(source, v, Literal.TRUE); + } + + public Values(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); } private Values(StreamInput in) throws IOException { @@ -70,12 +76,17 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Values::new, field()); + return NodeInfo.create(this, Values::new, field(), filter()); } @Override public Values replaceChildren(List newChildren) { - return new Values(source(), newChildren.get(0)); + return new Values(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public Values withFilter(Expression filter) { + return new Values(source(), field(), filter); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java index 23a20d9897e72..dbcc50cea3b9b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java @@ -7,11 +7,13 @@ package org.elasticsearch.xpack.esql.expression.function.aggregate; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -54,21 +56,30 @@ public WeightedAvg( @Param(name = "number", type = { "double", "integer", "long" }, description = "A numeric value.") Expression field, @Param(name = "weight", type = { "double", "integer", "long" }, description = "A numeric weight.") Expression weight ) { - super(source, field, List.of(weight)); + this(source, field, Literal.TRUE, weight); + } + + public WeightedAvg(Source source, Expression field, Expression filter, Expression weight) { + super(source, field, filter, List.of(weight)); this.weight = weight; } private WeightedAvg(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteable(Expression.class) + : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) + : in.readNamedWriteable(Expression.class) + ); } @Override - public void writeTo(StreamOutput out) throws IOException { - source().writeTo(out); - List fields = children(); - assert fields.size() == 2; - out.writeNamedWriteable(fields.get(0)); - out.writeNamedWriteable(fields.get(1)); + protected void deprecatedWriteParams(StreamOutput out) throws IOException { + out.writeNamedWriteable(weight); } @Override @@ -121,12 +132,17 @@ public DataType dataType() { @Override protected NodeInfo info() { - return NodeInfo.create(this, WeightedAvg::new, field(), weight); + return NodeInfo.create(this, WeightedAvg::new, field(), filter(), weight); } @Override public WeightedAvg replaceChildren(List newChildren) { - return new WeightedAvg(source(), newChildren.get(0), newChildren.get(1)); + return new WeightedAvg(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } + + @Override + public WeightedAvg withFilter(Expression filter) { + return new WeightedAvg(source(), field(), filter, weight()); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 54730eec4f317..2f97de4c64469 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -7,22 +7,18 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; -import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import static java.util.Collections.singletonList; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -34,47 +30,79 @@ */ public abstract class FullTextFunction extends Function { public static List getNamedWriteables() { - List entries = new ArrayList<>(); - if (EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()) { - entries.add(QueryStringFunction.ENTRY); - } - return entries; + return List.of(QueryString.ENTRY, Match.ENTRY); } private final Expression query; - protected FullTextFunction(Source source, Expression query) { - super(source, singletonList(query)); + protected FullTextFunction(Source source, Expression query, List children) { + super(source, children); this.query = query; } - protected FullTextFunction(StreamInput in) throws IOException { - this(Source.readFrom((StreamInput & PlanStreamInput) in), in.readNamedWriteable(Expression.class)); - } - @Override public DataType dataType() { return DataType.BOOLEAN; } @Override - protected TypeResolution resolveType() { + protected final TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } - return isString(query(), sourceText(), DEFAULT).and(isNotNullAndFoldable(query(), functionName(), DEFAULT)); + return resolveNonQueryParamTypes().and(resolveQueryParamType()); + } + + /** + * Resolves the type for the query parameter, as part of the type resolution for the function + * + * @return type resolution for query parameter + */ + private TypeResolution resolveQueryParamType() { + return isString(query(), sourceText(), queryParamOrdinal()).and(isNotNullAndFoldable(query(), sourceText(), queryParamOrdinal())); + } + + /** + * Subclasses can override this method for custom type resolution for additional function parameters + * + * @return type resolution for non-query parameter types + */ + protected TypeResolution resolveNonQueryParamTypes() { + return TypeResolution.TYPE_RESOLVED; } public Expression query() { return query; } - @Override - public void writeTo(StreamOutput out) throws IOException { - source().writeTo(out); - out.writeNamedWriteable(query); + /** + * Returns the resulting query as a String + * + * @return query expression as a string + */ + public final String queryAsText() { + Object queryAsObject = query().fold(); + if (queryAsObject instanceof BytesRef bytesRef) { + return bytesRef.utf8ToString(); + } + + throw new IllegalArgumentException( + format(null, "{} argument in {} function needs to be resolved to a string", queryParamOrdinal(), functionName()) + ); } - public abstract Query asQuery(); + /** + * Returns the param ordinal for the query parameter so it can be used in error messages + * + * @return Query ordinal for the + */ + protected TypeResolutions.ParamOrdinal queryParamOrdinal() { + return DEFAULT; + } + + @Override + public Nullability nullable() { + return Nullability.FALSE; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java new file mode 100644 index 0000000000000..b4e0f3c743216 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.common.Failure; +import org.elasticsearch.xpack.esql.common.Failures; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +/** + * Full text function that performs a {@link QueryStringQuery} . + */ +public class Match extends FullTextFunction implements Validatable { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Match", Match::new); + + private final Expression field; + + @FunctionInfo( + returnType = "boolean", + preview = true, + description = "Performs a match query on the specified field. Returns true if the provided query matches the row.", + examples = { @Example(file = "match-function", tag = "match-with-field") } + ) + public Match( + Source source, + @Param(name = "field", type = { "keyword", "text" }, description = "Field that the query will target.") Expression field, + @Param( + name = "query", + type = { "keyword", "text" }, + description = "Text you wish to find in the provided field." + ) Expression matchQuery + ) { + super(source, matchQuery, List.of(field, matchQuery)); + this.field = field; + } + + private Match(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(field); + out.writeNamedWriteable(query()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveNonQueryParamTypes() { + return isNotNull(field, sourceText(), FIRST).and(isString(field, sourceText(), FIRST)).and(super.resolveNonQueryParamTypes()); + } + + @Override + public void validate(Failures failures) { + if (field instanceof FieldAttribute == false) { + failures.add( + Failure.fail( + field, + "[{}] cannot operate on [{}], which is not a field from an index mapping", + functionName(), + field.sourceText() + ) + ); + } + } + + @Override + public Expression replaceChildren(List newChildren) { + // Query is the first child, field is the second child + return new Match(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Match::new, field, query()); + } + + protected TypeResolutions.ParamOrdinal queryParamOrdinal() { + return SECOND; + } + + public Expression field() { + return field; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java similarity index 66% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java index fa331acd08655..0d7d15a13dd80 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java @@ -7,32 +7,27 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; import java.util.List; -import java.util.Map; /** * Full text function that performs a {@link QueryStringQuery} . */ -public class QueryStringFunction extends FullTextFunction { +public class QueryString extends FullTextFunction { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( - Expression.class, - "QStr", - QueryStringFunction::new - ); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "QStr", QueryString::new); @FunctionInfo( returnType = "boolean", @@ -40,7 +35,7 @@ public class QueryStringFunction extends FullTextFunction { description = "Performs a query string query. Returns true if the provided query string matches the row.", examples = { @Example(file = "qstr-function", tag = "qstr-with-field") } ) - public QueryStringFunction( + public QueryString( Source source, @Param( name = "query", @@ -48,40 +43,37 @@ public QueryStringFunction( description = "Query string in Lucene query string format." ) Expression queryString ) { - super(source, queryString); + super(source, queryString, List.of(queryString)); } - private QueryStringFunction(StreamInput in) throws IOException { - super(in); + private QueryString(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); } @Override - public String functionName() { - return "QSTR"; + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(query()); } @Override - public Query asQuery() { - Object queryAsObject = query().fold(); - if (queryAsObject instanceof BytesRef queryAsBytesRef) { - return new QueryStringQuery(source(), queryAsBytesRef.utf8ToString(), Map.of(), null); - } else { - throw new IllegalArgumentException("Query in QSTR needs to be resolved to a string"); - } + public String getWriteableName() { + return ENTRY.name; } @Override - public Expression replaceChildren(List newChildren) { - return new QueryStringFunction(source(), newChildren.get(0)); + public String functionName() { + return "QSTR"; } @Override - protected NodeInfo info() { - return NodeInfo.create(this, QueryStringFunction::new, query()); + public Expression replaceChildren(List newChildren) { + return new QueryString(source(), newChildren.get(0)); } @Override - public String getWriteableName() { - return ENTRY.name; + protected NodeInfo info() { + return NodeInfo.create(this, QueryString::new, query()); } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java index 5e8d39217fcca..8839244e6c601 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java @@ -28,6 +28,8 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.isNull; @@ -203,7 +205,7 @@ public void setCrsType(DataType dataType) { } private static final String[] GEO_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() }; - private static final String[] CARTESIAN_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() }; + private static final String[] CARTESIAN_TYPE_NAMES = new String[] { CARTESIAN_POINT.typeName(), CARTESIAN_SHAPE.typeName() }; protected static boolean spatialCRSCompatible(DataType spatialDataType, DataType otherDataType) { return DataType.isSpatialGeo(spatialDataType) && DataType.isSpatialGeo(otherDataType) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index bfbf5a8f0c66f..a1da269f896da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -53,6 +53,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.SkipQueryOnEmptyMappings; import org.elasticsearch.xpack.esql.optimizer.rules.logical.SkipQueryOnLimitZero; import org.elasticsearch.xpack.esql.optimizer.rules.logical.SplitInWithFoldableValue; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteFilteredExpression; import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSpatialSurrogates; import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSurrogates; import org.elasticsearch.xpack.esql.optimizer.rules.logical.TranslateMetricsAggregate; @@ -122,6 +123,9 @@ protected static Batch substitutions() { "Substitutions", Limiter.ONCE, new ReplaceLookupWithJoin(), + // translate filtered expressions into aggregate with filters - can't use surrogate expressions because it was + // retrofitted for constant folding - this needs to be fixed + new SubstituteFilteredExpression(), new RemoveStatsOverride(), // first extract nested expressions inside aggs new ReplaceStatsNestedExpressionWithEval(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java index 0561865213a1b..0f08cd66444a3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; public class FoldNull extends OptimizerRules.OptimizerExpressionRule { @@ -23,6 +24,16 @@ public FoldNull() { @Override public Expression rule(Expression e) { Expression result = tryReplaceIsNullIsNotNull(e); + + // convert an aggregate null filter into a false + // perform this early to prevent the rule from converting the null filter into nullifying the whole expression + // P.S. this could be done inside the Aggregate but this place better centralizes the logic + if (e instanceof AggregateFunction agg) { + if (Expressions.isNull(agg.filter())) { + return agg.withFilter(Literal.of(agg.filter(), false)); + } + } + if (result != e) { return result; } else if (e instanceof In in) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java index d74811518624a..559546d48eb7d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsAggExpressionWithEval.java @@ -13,7 +13,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; @@ -25,8 +24,6 @@ import java.util.List; import java.util.Map; -import static java.util.Collections.singleton; - /** * Replace nested expressions over aggregates with synthetic eval post the aggregation * stats a = sum(a) + min(b) by x @@ -71,16 +68,13 @@ protected LogicalPlan rule(Aggregate aggregate) { for (NamedExpression agg : aggs) { if (agg instanceof Alias as) { - // if the child a nested expression + // use intermediate variable to mark child as final for lambda use Expression child = as.child(); // common case - handle duplicates if (child instanceof AggregateFunction af) { - AggregateFunction canonical = (AggregateFunction) af.canonical(); - Expression field = canonical.field().transformUp(e -> aliases.resolve(e, e)); - canonical = (AggregateFunction) canonical.replaceChildren( - CollectionUtils.combine(singleton(field), canonical.parameters()) - ); + // canonical representation, with resolved aliases + AggregateFunction canonical = (AggregateFunction) af.canonical().transformUp(e -> aliases.resolve(e, e)); Alias found = rootAggs.get(canonical); // aggregate is new @@ -130,7 +124,7 @@ protected LogicalPlan rule(Aggregate aggregate) { LogicalPlan plan = aggregate; if (changed.get()) { Source source = aggregate.source(); - plan = new Aggregate(source, aggregate.child(), aggregate.aggregateType(), aggregate.groupings(), newAggs); + plan = aggregate.with(aggregate.child(), aggregate.groupings(), newAggs); if (newEvals.size() > 0) { plan = new Eval(source, plan, newEvals); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java new file mode 100644 index 0000000000000..c8369d2b08a34 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.logical; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.OptimizerExpressionRule; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection; + +/** + * This rule should not be needed - the substitute infrastructure should be enough. + */ +public class SubstituteFilteredExpression extends OptimizerExpressionRule { + public SubstituteFilteredExpression() { + super(TransformDirection.UP); + } + + @Override + protected Expression rule(FilteredExpression filteredExpression) { + return filteredExpression.surrogate(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index be6e124502ba5..ec25c69deba5c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -216,31 +216,40 @@ private Expression rewriteDistanceFilter( Number number, ComparisonType comparisonType ) { + DataType shapeDataType = getShapeDataType(spatialExp); Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(literalExp); if (geometry instanceof Point point) { double distance = number.doubleValue(); Source source = comparison.source(); if (comparisonType.lt) { distance = comparisonType.eq ? distance : Math.nextDown(distance); - return new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp)); + return new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp, shapeDataType)); } else if (comparisonType.gt) { distance = comparisonType.eq ? distance : Math.nextUp(distance); - return new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, distance, literalExp)); + return new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, distance, literalExp, shapeDataType)); } else if (comparisonType.eq) { return new And( source, - new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp)), - new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, Math.nextDown(distance), literalExp)) + new SpatialIntersects(source, spatialExp, makeCircleLiteral(point, distance, literalExp, shapeDataType)), + new SpatialDisjoint(source, spatialExp, makeCircleLiteral(point, Math.nextDown(distance), literalExp, shapeDataType)) ); } } return comparison; } - private Literal makeCircleLiteral(Point point, double distance, Expression literalExpression) { + private Literal makeCircleLiteral(Point point, double distance, Expression literalExpression, DataType shapeDataType) { var circle = new Circle(point.getX(), point.getY(), distance); var wkb = WellKnownBinary.toWKB(circle, ByteOrder.LITTLE_ENDIAN); - return new Literal(literalExpression.source(), new BytesRef(wkb), DataType.GEO_SHAPE); + return new Literal(literalExpression.source(), new BytesRef(wkb), shapeDataType); + } + + private DataType getShapeDataType(Expression expression) { + return switch (expression.dataType()) { + case GEO_POINT, GEO_SHAPE -> DataType.GEO_SHAPE; + case CARTESIAN_POINT, CARTESIAN_SHAPE -> DataType.CARTESIAN_SHAPE; + default -> throw new IllegalArgumentException("Unsupported spatial data type: " + expression.dataType()); + }; } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 1ba966e318219..2209dffe5af06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -32,7 +32,8 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Queries; -import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; @@ -250,8 +251,10 @@ public static boolean canPushToSource(Expression exp, Predicate return mqp.field() instanceof FieldAttribute && DataType.isString(mqp.field().dataType()); } else if (exp instanceof StringQueryPredicate) { return true; - } else if (exp instanceof FullTextFunction) { + } else if (exp instanceof QueryString) { return true; + } else if (exp instanceof Match mf) { + return mf.field() instanceof FieldAttribute && DataType.isString(mf.field().dataType()); } return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java index 6db35fa0a06d6..855faf9df5ed2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java @@ -24,7 +24,6 @@ import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.EvalExec; -import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; @@ -82,17 +81,6 @@ public PhysicalPlan rewrite(TopNExec topNExec) { } } - /** - * TODO: Consider deleting this case entirely. We do not know if this is ever hit. - */ - record PushableExchangeExec(ExchangeExec exchangeExec, EsQueryExec queryExec) implements Pushable { - public PhysicalPlan rewrite(TopNExec topNExec) { - var sorts = buildFieldSorts(topNExec.order()); - var limit = topNExec.limit(); - return exchangeExec.replaceChild(queryExec.withSorts(sorts).withLimit(limit)); - } - } - record PushableQueryExec(EsQueryExec queryExec) implements Pushable { public PhysicalPlan rewrite(TopNExec topNExec) { var sorts = buildFieldSorts(topNExec.order()); @@ -141,13 +129,6 @@ && canPushDownOrders(topNExec.order(), hasIdenticalDelegate)) { // With the simplest case of `FROM index | SORT ...` we only allow pushing down if the sort is on a field return new PushableQueryExec(queryExec); } - if (child instanceof ExchangeExec exchangeExec - && exchangeExec.child() instanceof EsQueryExec queryExec - && queryExec.canPushSorts() - && canPushDownOrders(topNExec.order(), hasIdenticalDelegate)) { - // When we have an exchange between the FROM and the SORT, we also only allow pushing down if the sort is on a field - return new PushableExchangeExec(exchangeExec, queryExec); - } if (child instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec queryExec && queryExec.canPushSorts()) { // When we have an EVAL between the FROM and the SORT, we consider pushing down if the sort is on a field and/or // a distance function defined in the EVAL. We also move the EVAL to after the SORT. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index b5ca44826c051..aa6ddfb433d23 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -23,7 +23,6 @@ null null null null -null '|' null null @@ -63,6 +62,7 @@ null '*' '/' '%' +'match' null null ']' @@ -141,7 +141,6 @@ STATS WHERE DEV_INLINESTATS DEV_LOOKUP -DEV_MATCH DEV_METRICS UNKNOWN_CMD LINE_COMMENT @@ -186,6 +185,7 @@ MINUS ASTERISK SLASH PERCENT +MATCH NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET @@ -263,7 +263,6 @@ STATS WHERE DEV_INLINESTATS DEV_LOOKUP -DEV_MATCH DEV_METRICS UNKNOWN_CMD LINE_COMMENT @@ -318,7 +317,8 @@ MINUS ASTERISK SLASH PERCENT -DEV_MATCH_OP +MATCH +NESTED_WHERE NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET @@ -466,4 +466,4 @@ METRICS_MODE CLOSING_METRICS_MODE atn: -[4, 0, 120, 1475, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 587, 8, 20, 11, 20, 12, 20, 588, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 597, 8, 21, 10, 21, 12, 21, 600, 9, 21, 1, 21, 3, 21, 603, 8, 21, 1, 21, 3, 21, 606, 8, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 615, 8, 22, 10, 22, 12, 22, 618, 9, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 4, 23, 626, 8, 23, 11, 23, 12, 23, 627, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 3, 29, 647, 8, 29, 1, 29, 4, 29, 650, 8, 29, 11, 29, 12, 29, 651, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 32, 3, 32, 661, 8, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 3, 34, 668, 8, 34, 1, 35, 1, 35, 1, 35, 5, 35, 673, 8, 35, 10, 35, 12, 35, 676, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 684, 8, 35, 10, 35, 12, 35, 687, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 3, 35, 694, 8, 35, 1, 35, 3, 35, 697, 8, 35, 3, 35, 699, 8, 35, 1, 36, 4, 36, 702, 8, 36, 11, 36, 12, 36, 703, 1, 37, 4, 37, 707, 8, 37, 11, 37, 12, 37, 708, 1, 37, 1, 37, 5, 37, 713, 8, 37, 10, 37, 12, 37, 716, 9, 37, 1, 37, 1, 37, 4, 37, 720, 8, 37, 11, 37, 12, 37, 721, 1, 37, 4, 37, 725, 8, 37, 11, 37, 12, 37, 726, 1, 37, 1, 37, 5, 37, 731, 8, 37, 10, 37, 12, 37, 734, 9, 37, 3, 37, 736, 8, 37, 1, 37, 1, 37, 1, 37, 1, 37, 4, 37, 742, 8, 37, 11, 37, 12, 37, 743, 1, 37, 1, 37, 3, 37, 748, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 875, 8, 74, 1, 74, 5, 74, 878, 8, 74, 10, 74, 12, 74, 881, 9, 74, 1, 74, 1, 74, 4, 74, 885, 8, 74, 11, 74, 12, 74, 886, 3, 74, 889, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 903, 8, 77, 10, 77, 12, 77, 906, 9, 77, 1, 77, 1, 77, 3, 77, 910, 8, 77, 1, 77, 4, 77, 913, 8, 77, 11, 77, 12, 77, 914, 3, 77, 917, 8, 77, 1, 78, 1, 78, 4, 78, 921, 8, 78, 11, 78, 12, 78, 922, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 1000, 8, 95, 1, 96, 4, 96, 1003, 8, 96, 11, 96, 12, 96, 1004, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 3, 107, 1052, 8, 107, 1, 108, 1, 108, 3, 108, 1056, 8, 108, 1, 108, 5, 108, 1059, 8, 108, 10, 108, 12, 108, 1062, 9, 108, 1, 108, 1, 108, 3, 108, 1066, 8, 108, 1, 108, 4, 108, 1069, 8, 108, 11, 108, 12, 108, 1070, 3, 108, 1073, 8, 108, 1, 109, 1, 109, 4, 109, 1077, 8, 109, 11, 109, 12, 109, 1078, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 129, 4, 129, 1162, 8, 129, 11, 129, 12, 129, 1163, 1, 129, 1, 129, 3, 129, 1168, 8, 129, 1, 129, 4, 129, 1171, 8, 129, 11, 129, 12, 129, 1172, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 4, 162, 1312, 8, 162, 11, 162, 12, 162, 1313, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 2, 616, 685, 0, 198, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 25, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 0, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 0, 233, 80, 235, 81, 237, 82, 239, 83, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 84, 255, 0, 257, 85, 259, 86, 261, 87, 263, 0, 265, 0, 267, 88, 269, 89, 271, 0, 273, 90, 275, 0, 277, 91, 279, 92, 281, 93, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 0, 301, 94, 303, 95, 305, 96, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 97, 321, 98, 323, 99, 325, 0, 327, 100, 329, 101, 331, 102, 333, 103, 335, 0, 337, 104, 339, 105, 341, 106, 343, 107, 345, 108, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 0, 361, 109, 363, 110, 365, 111, 367, 0, 369, 0, 371, 0, 373, 0, 375, 112, 377, 113, 379, 114, 381, 0, 383, 0, 385, 0, 387, 115, 389, 116, 391, 117, 393, 0, 395, 0, 397, 118, 399, 119, 401, 120, 403, 0, 405, 0, 407, 0, 409, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1503, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 1, 63, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 227, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 9, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 10, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 12, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 13, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 14, 409, 1, 0, 0, 0, 15, 411, 1, 0, 0, 0, 17, 421, 1, 0, 0, 0, 19, 428, 1, 0, 0, 0, 21, 437, 1, 0, 0, 0, 23, 444, 1, 0, 0, 0, 25, 454, 1, 0, 0, 0, 27, 461, 1, 0, 0, 0, 29, 468, 1, 0, 0, 0, 31, 475, 1, 0, 0, 0, 33, 483, 1, 0, 0, 0, 35, 495, 1, 0, 0, 0, 37, 504, 1, 0, 0, 0, 39, 510, 1, 0, 0, 0, 41, 517, 1, 0, 0, 0, 43, 524, 1, 0, 0, 0, 45, 532, 1, 0, 0, 0, 47, 540, 1, 0, 0, 0, 49, 555, 1, 0, 0, 0, 51, 565, 1, 0, 0, 0, 53, 574, 1, 0, 0, 0, 55, 586, 1, 0, 0, 0, 57, 592, 1, 0, 0, 0, 59, 609, 1, 0, 0, 0, 61, 625, 1, 0, 0, 0, 63, 631, 1, 0, 0, 0, 65, 635, 1, 0, 0, 0, 67, 637, 1, 0, 0, 0, 69, 639, 1, 0, 0, 0, 71, 642, 1, 0, 0, 0, 73, 644, 1, 0, 0, 0, 75, 653, 1, 0, 0, 0, 77, 655, 1, 0, 0, 0, 79, 660, 1, 0, 0, 0, 81, 662, 1, 0, 0, 0, 83, 667, 1, 0, 0, 0, 85, 698, 1, 0, 0, 0, 87, 701, 1, 0, 0, 0, 89, 747, 1, 0, 0, 0, 91, 749, 1, 0, 0, 0, 93, 752, 1, 0, 0, 0, 95, 756, 1, 0, 0, 0, 97, 760, 1, 0, 0, 0, 99, 762, 1, 0, 0, 0, 101, 765, 1, 0, 0, 0, 103, 767, 1, 0, 0, 0, 105, 772, 1, 0, 0, 0, 107, 774, 1, 0, 0, 0, 109, 780, 1, 0, 0, 0, 111, 786, 1, 0, 0, 0, 113, 789, 1, 0, 0, 0, 115, 792, 1, 0, 0, 0, 117, 797, 1, 0, 0, 0, 119, 802, 1, 0, 0, 0, 121, 804, 1, 0, 0, 0, 123, 808, 1, 0, 0, 0, 125, 813, 1, 0, 0, 0, 127, 819, 1, 0, 0, 0, 129, 822, 1, 0, 0, 0, 131, 824, 1, 0, 0, 0, 133, 830, 1, 0, 0, 0, 135, 832, 1, 0, 0, 0, 137, 837, 1, 0, 0, 0, 139, 840, 1, 0, 0, 0, 141, 843, 1, 0, 0, 0, 143, 846, 1, 0, 0, 0, 145, 848, 1, 0, 0, 0, 147, 851, 1, 0, 0, 0, 149, 853, 1, 0, 0, 0, 151, 856, 1, 0, 0, 0, 153, 858, 1, 0, 0, 0, 155, 860, 1, 0, 0, 0, 157, 862, 1, 0, 0, 0, 159, 864, 1, 0, 0, 0, 161, 866, 1, 0, 0, 0, 163, 888, 1, 0, 0, 0, 165, 890, 1, 0, 0, 0, 167, 895, 1, 0, 0, 0, 169, 916, 1, 0, 0, 0, 171, 918, 1, 0, 0, 0, 173, 926, 1, 0, 0, 0, 175, 928, 1, 0, 0, 0, 177, 932, 1, 0, 0, 0, 179, 936, 1, 0, 0, 0, 181, 940, 1, 0, 0, 0, 183, 945, 1, 0, 0, 0, 185, 950, 1, 0, 0, 0, 187, 954, 1, 0, 0, 0, 189, 958, 1, 0, 0, 0, 191, 962, 1, 0, 0, 0, 193, 967, 1, 0, 0, 0, 195, 971, 1, 0, 0, 0, 197, 975, 1, 0, 0, 0, 199, 979, 1, 0, 0, 0, 201, 983, 1, 0, 0, 0, 203, 987, 1, 0, 0, 0, 205, 999, 1, 0, 0, 0, 207, 1002, 1, 0, 0, 0, 209, 1006, 1, 0, 0, 0, 211, 1010, 1, 0, 0, 0, 213, 1014, 1, 0, 0, 0, 215, 1018, 1, 0, 0, 0, 217, 1022, 1, 0, 0, 0, 219, 1026, 1, 0, 0, 0, 221, 1031, 1, 0, 0, 0, 223, 1035, 1, 0, 0, 0, 225, 1039, 1, 0, 0, 0, 227, 1043, 1, 0, 0, 0, 229, 1051, 1, 0, 0, 0, 231, 1072, 1, 0, 0, 0, 233, 1076, 1, 0, 0, 0, 235, 1080, 1, 0, 0, 0, 237, 1084, 1, 0, 0, 0, 239, 1088, 1, 0, 0, 0, 241, 1092, 1, 0, 0, 0, 243, 1097, 1, 0, 0, 0, 245, 1101, 1, 0, 0, 0, 247, 1105, 1, 0, 0, 0, 249, 1109, 1, 0, 0, 0, 251, 1113, 1, 0, 0, 0, 253, 1117, 1, 0, 0, 0, 255, 1120, 1, 0, 0, 0, 257, 1124, 1, 0, 0, 0, 259, 1128, 1, 0, 0, 0, 261, 1132, 1, 0, 0, 0, 263, 1136, 1, 0, 0, 0, 265, 1141, 1, 0, 0, 0, 267, 1146, 1, 0, 0, 0, 269, 1151, 1, 0, 0, 0, 271, 1158, 1, 0, 0, 0, 273, 1167, 1, 0, 0, 0, 275, 1174, 1, 0, 0, 0, 277, 1178, 1, 0, 0, 0, 279, 1182, 1, 0, 0, 0, 281, 1186, 1, 0, 0, 0, 283, 1190, 1, 0, 0, 0, 285, 1196, 1, 0, 0, 0, 287, 1200, 1, 0, 0, 0, 289, 1204, 1, 0, 0, 0, 291, 1208, 1, 0, 0, 0, 293, 1212, 1, 0, 0, 0, 295, 1216, 1, 0, 0, 0, 297, 1220, 1, 0, 0, 0, 299, 1224, 1, 0, 0, 0, 301, 1228, 1, 0, 0, 0, 303, 1232, 1, 0, 0, 0, 305, 1236, 1, 0, 0, 0, 307, 1240, 1, 0, 0, 0, 309, 1245, 1, 0, 0, 0, 311, 1249, 1, 0, 0, 0, 313, 1253, 1, 0, 0, 0, 315, 1257, 1, 0, 0, 0, 317, 1261, 1, 0, 0, 0, 319, 1265, 1, 0, 0, 0, 321, 1269, 1, 0, 0, 0, 323, 1273, 1, 0, 0, 0, 325, 1277, 1, 0, 0, 0, 327, 1282, 1, 0, 0, 0, 329, 1287, 1, 0, 0, 0, 331, 1291, 1, 0, 0, 0, 333, 1295, 1, 0, 0, 0, 335, 1299, 1, 0, 0, 0, 337, 1304, 1, 0, 0, 0, 339, 1311, 1, 0, 0, 0, 341, 1315, 1, 0, 0, 0, 343, 1319, 1, 0, 0, 0, 345, 1323, 1, 0, 0, 0, 347, 1327, 1, 0, 0, 0, 349, 1332, 1, 0, 0, 0, 351, 1336, 1, 0, 0, 0, 353, 1340, 1, 0, 0, 0, 355, 1344, 1, 0, 0, 0, 357, 1349, 1, 0, 0, 0, 359, 1353, 1, 0, 0, 0, 361, 1357, 1, 0, 0, 0, 363, 1361, 1, 0, 0, 0, 365, 1365, 1, 0, 0, 0, 367, 1369, 1, 0, 0, 0, 369, 1375, 1, 0, 0, 0, 371, 1379, 1, 0, 0, 0, 373, 1383, 1, 0, 0, 0, 375, 1387, 1, 0, 0, 0, 377, 1391, 1, 0, 0, 0, 379, 1395, 1, 0, 0, 0, 381, 1399, 1, 0, 0, 0, 383, 1404, 1, 0, 0, 0, 385, 1410, 1, 0, 0, 0, 387, 1416, 1, 0, 0, 0, 389, 1420, 1, 0, 0, 0, 391, 1424, 1, 0, 0, 0, 393, 1428, 1, 0, 0, 0, 395, 1434, 1, 0, 0, 0, 397, 1440, 1, 0, 0, 0, 399, 1444, 1, 0, 0, 0, 401, 1448, 1, 0, 0, 0, 403, 1452, 1, 0, 0, 0, 405, 1458, 1, 0, 0, 0, 407, 1464, 1, 0, 0, 0, 409, 1470, 1, 0, 0, 0, 411, 412, 7, 0, 0, 0, 412, 413, 7, 1, 0, 0, 413, 414, 7, 2, 0, 0, 414, 415, 7, 2, 0, 0, 415, 416, 7, 3, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 5, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 0, 0, 0, 420, 16, 1, 0, 0, 0, 421, 422, 7, 0, 0, 0, 422, 423, 7, 6, 0, 0, 423, 424, 7, 7, 0, 0, 424, 425, 7, 8, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 1, 1, 0, 427, 18, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 9, 0, 0, 430, 431, 7, 6, 0, 0, 431, 432, 7, 1, 0, 0, 432, 433, 7, 4, 0, 0, 433, 434, 7, 10, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 2, 2, 0, 436, 20, 1, 0, 0, 0, 437, 438, 7, 3, 0, 0, 438, 439, 7, 11, 0, 0, 439, 440, 7, 12, 0, 0, 440, 441, 7, 13, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 3, 0, 0, 443, 22, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 446, 7, 14, 0, 0, 446, 447, 7, 8, 0, 0, 447, 448, 7, 13, 0, 0, 448, 449, 7, 12, 0, 0, 449, 450, 7, 1, 0, 0, 450, 451, 7, 9, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 4, 3, 0, 453, 24, 1, 0, 0, 0, 454, 455, 7, 15, 0, 0, 455, 456, 7, 6, 0, 0, 456, 457, 7, 7, 0, 0, 457, 458, 7, 16, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 5, 4, 0, 460, 26, 1, 0, 0, 0, 461, 462, 7, 17, 0, 0, 462, 463, 7, 6, 0, 0, 463, 464, 7, 7, 0, 0, 464, 465, 7, 18, 0, 0, 465, 466, 1, 0, 0, 0, 466, 467, 6, 6, 0, 0, 467, 28, 1, 0, 0, 0, 468, 469, 7, 18, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 8, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 1, 0, 474, 30, 1, 0, 0, 0, 475, 476, 7, 13, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 16, 0, 0, 478, 479, 7, 1, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 8, 0, 0, 482, 32, 1, 0, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 11, 0, 0, 485, 486, 5, 95, 0, 0, 486, 487, 7, 3, 0, 0, 487, 488, 7, 14, 0, 0, 488, 489, 7, 8, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 9, 0, 0, 491, 492, 7, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 9, 5, 0, 494, 34, 1, 0, 0, 0, 495, 496, 7, 6, 0, 0, 496, 497, 7, 3, 0, 0, 497, 498, 7, 9, 0, 0, 498, 499, 7, 12, 0, 0, 499, 500, 7, 16, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 10, 6, 0, 503, 36, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 506, 7, 7, 0, 0, 506, 507, 7, 19, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 11, 0, 0, 509, 38, 1, 0, 0, 0, 510, 511, 7, 2, 0, 0, 511, 512, 7, 10, 0, 0, 512, 513, 7, 7, 0, 0, 513, 514, 7, 19, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 12, 7, 0, 516, 40, 1, 0, 0, 0, 517, 518, 7, 2, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 13, 0, 0, 523, 42, 1, 0, 0, 0, 524, 525, 7, 2, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 12, 0, 0, 527, 528, 7, 5, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 14, 0, 0, 531, 44, 1, 0, 0, 0, 532, 533, 7, 19, 0, 0, 533, 534, 7, 10, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 7, 6, 0, 0, 536, 537, 7, 3, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 15, 0, 0, 539, 46, 1, 0, 0, 0, 540, 541, 4, 16, 0, 0, 541, 542, 7, 1, 0, 0, 542, 543, 7, 9, 0, 0, 543, 544, 7, 13, 0, 0, 544, 545, 7, 1, 0, 0, 545, 546, 7, 9, 0, 0, 546, 547, 7, 3, 0, 0, 547, 548, 7, 2, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 12, 0, 0, 550, 551, 7, 5, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 16, 0, 0, 554, 48, 1, 0, 0, 0, 555, 556, 4, 17, 1, 0, 556, 557, 7, 13, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 7, 0, 0, 559, 560, 7, 18, 0, 0, 560, 561, 7, 20, 0, 0, 561, 562, 7, 8, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 17, 8, 0, 564, 50, 1, 0, 0, 0, 565, 566, 4, 18, 2, 0, 566, 567, 7, 16, 0, 0, 567, 568, 7, 12, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 4, 0, 0, 570, 571, 7, 10, 0, 0, 571, 572, 1, 0, 0, 0, 572, 573, 6, 18, 0, 0, 573, 52, 1, 0, 0, 0, 574, 575, 4, 19, 3, 0, 575, 576, 7, 16, 0, 0, 576, 577, 7, 3, 0, 0, 577, 578, 7, 5, 0, 0, 578, 579, 7, 6, 0, 0, 579, 580, 7, 1, 0, 0, 580, 581, 7, 4, 0, 0, 581, 582, 7, 2, 0, 0, 582, 583, 1, 0, 0, 0, 583, 584, 6, 19, 9, 0, 584, 54, 1, 0, 0, 0, 585, 587, 8, 21, 0, 0, 586, 585, 1, 0, 0, 0, 587, 588, 1, 0, 0, 0, 588, 586, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 6, 20, 0, 0, 591, 56, 1, 0, 0, 0, 592, 593, 5, 47, 0, 0, 593, 594, 5, 47, 0, 0, 594, 598, 1, 0, 0, 0, 595, 597, 8, 22, 0, 0, 596, 595, 1, 0, 0, 0, 597, 600, 1, 0, 0, 0, 598, 596, 1, 0, 0, 0, 598, 599, 1, 0, 0, 0, 599, 602, 1, 0, 0, 0, 600, 598, 1, 0, 0, 0, 601, 603, 5, 13, 0, 0, 602, 601, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 605, 1, 0, 0, 0, 604, 606, 5, 10, 0, 0, 605, 604, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 608, 6, 21, 10, 0, 608, 58, 1, 0, 0, 0, 609, 610, 5, 47, 0, 0, 610, 611, 5, 42, 0, 0, 611, 616, 1, 0, 0, 0, 612, 615, 3, 59, 22, 0, 613, 615, 9, 0, 0, 0, 614, 612, 1, 0, 0, 0, 614, 613, 1, 0, 0, 0, 615, 618, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 616, 614, 1, 0, 0, 0, 617, 619, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 619, 620, 5, 42, 0, 0, 620, 621, 5, 47, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 6, 22, 10, 0, 623, 60, 1, 0, 0, 0, 624, 626, 7, 23, 0, 0, 625, 624, 1, 0, 0, 0, 626, 627, 1, 0, 0, 0, 627, 625, 1, 0, 0, 0, 627, 628, 1, 0, 0, 0, 628, 629, 1, 0, 0, 0, 629, 630, 6, 23, 10, 0, 630, 62, 1, 0, 0, 0, 631, 632, 5, 124, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 6, 24, 11, 0, 634, 64, 1, 0, 0, 0, 635, 636, 7, 24, 0, 0, 636, 66, 1, 0, 0, 0, 637, 638, 7, 25, 0, 0, 638, 68, 1, 0, 0, 0, 639, 640, 5, 92, 0, 0, 640, 641, 7, 26, 0, 0, 641, 70, 1, 0, 0, 0, 642, 643, 8, 27, 0, 0, 643, 72, 1, 0, 0, 0, 644, 646, 7, 3, 0, 0, 645, 647, 7, 28, 0, 0, 646, 645, 1, 0, 0, 0, 646, 647, 1, 0, 0, 0, 647, 649, 1, 0, 0, 0, 648, 650, 3, 65, 25, 0, 649, 648, 1, 0, 0, 0, 650, 651, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 651, 652, 1, 0, 0, 0, 652, 74, 1, 0, 0, 0, 653, 654, 5, 64, 0, 0, 654, 76, 1, 0, 0, 0, 655, 656, 5, 96, 0, 0, 656, 78, 1, 0, 0, 0, 657, 661, 8, 29, 0, 0, 658, 659, 5, 96, 0, 0, 659, 661, 5, 96, 0, 0, 660, 657, 1, 0, 0, 0, 660, 658, 1, 0, 0, 0, 661, 80, 1, 0, 0, 0, 662, 663, 5, 95, 0, 0, 663, 82, 1, 0, 0, 0, 664, 668, 3, 67, 26, 0, 665, 668, 3, 65, 25, 0, 666, 668, 3, 81, 33, 0, 667, 664, 1, 0, 0, 0, 667, 665, 1, 0, 0, 0, 667, 666, 1, 0, 0, 0, 668, 84, 1, 0, 0, 0, 669, 674, 5, 34, 0, 0, 670, 673, 3, 69, 27, 0, 671, 673, 3, 71, 28, 0, 672, 670, 1, 0, 0, 0, 672, 671, 1, 0, 0, 0, 673, 676, 1, 0, 0, 0, 674, 672, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 675, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 699, 5, 34, 0, 0, 678, 679, 5, 34, 0, 0, 679, 680, 5, 34, 0, 0, 680, 681, 5, 34, 0, 0, 681, 685, 1, 0, 0, 0, 682, 684, 8, 22, 0, 0, 683, 682, 1, 0, 0, 0, 684, 687, 1, 0, 0, 0, 685, 686, 1, 0, 0, 0, 685, 683, 1, 0, 0, 0, 686, 688, 1, 0, 0, 0, 687, 685, 1, 0, 0, 0, 688, 689, 5, 34, 0, 0, 689, 690, 5, 34, 0, 0, 690, 691, 5, 34, 0, 0, 691, 693, 1, 0, 0, 0, 692, 694, 5, 34, 0, 0, 693, 692, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 696, 1, 0, 0, 0, 695, 697, 5, 34, 0, 0, 696, 695, 1, 0, 0, 0, 696, 697, 1, 0, 0, 0, 697, 699, 1, 0, 0, 0, 698, 669, 1, 0, 0, 0, 698, 678, 1, 0, 0, 0, 699, 86, 1, 0, 0, 0, 700, 702, 3, 65, 25, 0, 701, 700, 1, 0, 0, 0, 702, 703, 1, 0, 0, 0, 703, 701, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 88, 1, 0, 0, 0, 705, 707, 3, 65, 25, 0, 706, 705, 1, 0, 0, 0, 707, 708, 1, 0, 0, 0, 708, 706, 1, 0, 0, 0, 708, 709, 1, 0, 0, 0, 709, 710, 1, 0, 0, 0, 710, 714, 3, 105, 45, 0, 711, 713, 3, 65, 25, 0, 712, 711, 1, 0, 0, 0, 713, 716, 1, 0, 0, 0, 714, 712, 1, 0, 0, 0, 714, 715, 1, 0, 0, 0, 715, 748, 1, 0, 0, 0, 716, 714, 1, 0, 0, 0, 717, 719, 3, 105, 45, 0, 718, 720, 3, 65, 25, 0, 719, 718, 1, 0, 0, 0, 720, 721, 1, 0, 0, 0, 721, 719, 1, 0, 0, 0, 721, 722, 1, 0, 0, 0, 722, 748, 1, 0, 0, 0, 723, 725, 3, 65, 25, 0, 724, 723, 1, 0, 0, 0, 725, 726, 1, 0, 0, 0, 726, 724, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 735, 1, 0, 0, 0, 728, 732, 3, 105, 45, 0, 729, 731, 3, 65, 25, 0, 730, 729, 1, 0, 0, 0, 731, 734, 1, 0, 0, 0, 732, 730, 1, 0, 0, 0, 732, 733, 1, 0, 0, 0, 733, 736, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 735, 728, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 738, 3, 73, 29, 0, 738, 748, 1, 0, 0, 0, 739, 741, 3, 105, 45, 0, 740, 742, 3, 65, 25, 0, 741, 740, 1, 0, 0, 0, 742, 743, 1, 0, 0, 0, 743, 741, 1, 0, 0, 0, 743, 744, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 746, 3, 73, 29, 0, 746, 748, 1, 0, 0, 0, 747, 706, 1, 0, 0, 0, 747, 717, 1, 0, 0, 0, 747, 724, 1, 0, 0, 0, 747, 739, 1, 0, 0, 0, 748, 90, 1, 0, 0, 0, 749, 750, 7, 30, 0, 0, 750, 751, 7, 31, 0, 0, 751, 92, 1, 0, 0, 0, 752, 753, 7, 12, 0, 0, 753, 754, 7, 9, 0, 0, 754, 755, 7, 0, 0, 0, 755, 94, 1, 0, 0, 0, 756, 757, 7, 12, 0, 0, 757, 758, 7, 2, 0, 0, 758, 759, 7, 4, 0, 0, 759, 96, 1, 0, 0, 0, 760, 761, 5, 61, 0, 0, 761, 98, 1, 0, 0, 0, 762, 763, 5, 58, 0, 0, 763, 764, 5, 58, 0, 0, 764, 100, 1, 0, 0, 0, 765, 766, 5, 44, 0, 0, 766, 102, 1, 0, 0, 0, 767, 768, 7, 0, 0, 0, 768, 769, 7, 3, 0, 0, 769, 770, 7, 2, 0, 0, 770, 771, 7, 4, 0, 0, 771, 104, 1, 0, 0, 0, 772, 773, 5, 46, 0, 0, 773, 106, 1, 0, 0, 0, 774, 775, 7, 15, 0, 0, 775, 776, 7, 12, 0, 0, 776, 777, 7, 13, 0, 0, 777, 778, 7, 2, 0, 0, 778, 779, 7, 3, 0, 0, 779, 108, 1, 0, 0, 0, 780, 781, 7, 15, 0, 0, 781, 782, 7, 1, 0, 0, 782, 783, 7, 6, 0, 0, 783, 784, 7, 2, 0, 0, 784, 785, 7, 5, 0, 0, 785, 110, 1, 0, 0, 0, 786, 787, 7, 1, 0, 0, 787, 788, 7, 9, 0, 0, 788, 112, 1, 0, 0, 0, 789, 790, 7, 1, 0, 0, 790, 791, 7, 2, 0, 0, 791, 114, 1, 0, 0, 0, 792, 793, 7, 13, 0, 0, 793, 794, 7, 12, 0, 0, 794, 795, 7, 2, 0, 0, 795, 796, 7, 5, 0, 0, 796, 116, 1, 0, 0, 0, 797, 798, 7, 13, 0, 0, 798, 799, 7, 1, 0, 0, 799, 800, 7, 18, 0, 0, 800, 801, 7, 3, 0, 0, 801, 118, 1, 0, 0, 0, 802, 803, 5, 40, 0, 0, 803, 120, 1, 0, 0, 0, 804, 805, 7, 9, 0, 0, 805, 806, 7, 7, 0, 0, 806, 807, 7, 5, 0, 0, 807, 122, 1, 0, 0, 0, 808, 809, 7, 9, 0, 0, 809, 810, 7, 20, 0, 0, 810, 811, 7, 13, 0, 0, 811, 812, 7, 13, 0, 0, 812, 124, 1, 0, 0, 0, 813, 814, 7, 9, 0, 0, 814, 815, 7, 20, 0, 0, 815, 816, 7, 13, 0, 0, 816, 817, 7, 13, 0, 0, 817, 818, 7, 2, 0, 0, 818, 126, 1, 0, 0, 0, 819, 820, 7, 7, 0, 0, 820, 821, 7, 6, 0, 0, 821, 128, 1, 0, 0, 0, 822, 823, 5, 63, 0, 0, 823, 130, 1, 0, 0, 0, 824, 825, 7, 6, 0, 0, 825, 826, 7, 13, 0, 0, 826, 827, 7, 1, 0, 0, 827, 828, 7, 18, 0, 0, 828, 829, 7, 3, 0, 0, 829, 132, 1, 0, 0, 0, 830, 831, 5, 41, 0, 0, 831, 134, 1, 0, 0, 0, 832, 833, 7, 5, 0, 0, 833, 834, 7, 6, 0, 0, 834, 835, 7, 20, 0, 0, 835, 836, 7, 3, 0, 0, 836, 136, 1, 0, 0, 0, 837, 838, 5, 61, 0, 0, 838, 839, 5, 61, 0, 0, 839, 138, 1, 0, 0, 0, 840, 841, 5, 61, 0, 0, 841, 842, 5, 126, 0, 0, 842, 140, 1, 0, 0, 0, 843, 844, 5, 33, 0, 0, 844, 845, 5, 61, 0, 0, 845, 142, 1, 0, 0, 0, 846, 847, 5, 60, 0, 0, 847, 144, 1, 0, 0, 0, 848, 849, 5, 60, 0, 0, 849, 850, 5, 61, 0, 0, 850, 146, 1, 0, 0, 0, 851, 852, 5, 62, 0, 0, 852, 148, 1, 0, 0, 0, 853, 854, 5, 62, 0, 0, 854, 855, 5, 61, 0, 0, 855, 150, 1, 0, 0, 0, 856, 857, 5, 43, 0, 0, 857, 152, 1, 0, 0, 0, 858, 859, 5, 45, 0, 0, 859, 154, 1, 0, 0, 0, 860, 861, 5, 42, 0, 0, 861, 156, 1, 0, 0, 0, 862, 863, 5, 47, 0, 0, 863, 158, 1, 0, 0, 0, 864, 865, 5, 37, 0, 0, 865, 160, 1, 0, 0, 0, 866, 867, 4, 73, 4, 0, 867, 868, 3, 51, 18, 0, 868, 869, 1, 0, 0, 0, 869, 870, 6, 73, 12, 0, 870, 162, 1, 0, 0, 0, 871, 874, 3, 129, 57, 0, 872, 875, 3, 67, 26, 0, 873, 875, 3, 81, 33, 0, 874, 872, 1, 0, 0, 0, 874, 873, 1, 0, 0, 0, 875, 879, 1, 0, 0, 0, 876, 878, 3, 83, 34, 0, 877, 876, 1, 0, 0, 0, 878, 881, 1, 0, 0, 0, 879, 877, 1, 0, 0, 0, 879, 880, 1, 0, 0, 0, 880, 889, 1, 0, 0, 0, 881, 879, 1, 0, 0, 0, 882, 884, 3, 129, 57, 0, 883, 885, 3, 65, 25, 0, 884, 883, 1, 0, 0, 0, 885, 886, 1, 0, 0, 0, 886, 884, 1, 0, 0, 0, 886, 887, 1, 0, 0, 0, 887, 889, 1, 0, 0, 0, 888, 871, 1, 0, 0, 0, 888, 882, 1, 0, 0, 0, 889, 164, 1, 0, 0, 0, 890, 891, 5, 91, 0, 0, 891, 892, 1, 0, 0, 0, 892, 893, 6, 75, 0, 0, 893, 894, 6, 75, 0, 0, 894, 166, 1, 0, 0, 0, 895, 896, 5, 93, 0, 0, 896, 897, 1, 0, 0, 0, 897, 898, 6, 76, 11, 0, 898, 899, 6, 76, 11, 0, 899, 168, 1, 0, 0, 0, 900, 904, 3, 67, 26, 0, 901, 903, 3, 83, 34, 0, 902, 901, 1, 0, 0, 0, 903, 906, 1, 0, 0, 0, 904, 902, 1, 0, 0, 0, 904, 905, 1, 0, 0, 0, 905, 917, 1, 0, 0, 0, 906, 904, 1, 0, 0, 0, 907, 910, 3, 81, 33, 0, 908, 910, 3, 75, 30, 0, 909, 907, 1, 0, 0, 0, 909, 908, 1, 0, 0, 0, 910, 912, 1, 0, 0, 0, 911, 913, 3, 83, 34, 0, 912, 911, 1, 0, 0, 0, 913, 914, 1, 0, 0, 0, 914, 912, 1, 0, 0, 0, 914, 915, 1, 0, 0, 0, 915, 917, 1, 0, 0, 0, 916, 900, 1, 0, 0, 0, 916, 909, 1, 0, 0, 0, 917, 170, 1, 0, 0, 0, 918, 920, 3, 77, 31, 0, 919, 921, 3, 79, 32, 0, 920, 919, 1, 0, 0, 0, 921, 922, 1, 0, 0, 0, 922, 920, 1, 0, 0, 0, 922, 923, 1, 0, 0, 0, 923, 924, 1, 0, 0, 0, 924, 925, 3, 77, 31, 0, 925, 172, 1, 0, 0, 0, 926, 927, 3, 171, 78, 0, 927, 174, 1, 0, 0, 0, 928, 929, 3, 57, 21, 0, 929, 930, 1, 0, 0, 0, 930, 931, 6, 80, 10, 0, 931, 176, 1, 0, 0, 0, 932, 933, 3, 59, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 81, 10, 0, 935, 178, 1, 0, 0, 0, 936, 937, 3, 61, 23, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 82, 10, 0, 939, 180, 1, 0, 0, 0, 940, 941, 3, 165, 75, 0, 941, 942, 1, 0, 0, 0, 942, 943, 6, 83, 13, 0, 943, 944, 6, 83, 14, 0, 944, 182, 1, 0, 0, 0, 945, 946, 3, 63, 24, 0, 946, 947, 1, 0, 0, 0, 947, 948, 6, 84, 15, 0, 948, 949, 6, 84, 11, 0, 949, 184, 1, 0, 0, 0, 950, 951, 3, 61, 23, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 85, 10, 0, 953, 186, 1, 0, 0, 0, 954, 955, 3, 57, 21, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 86, 10, 0, 957, 188, 1, 0, 0, 0, 958, 959, 3, 59, 22, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 87, 10, 0, 961, 190, 1, 0, 0, 0, 962, 963, 3, 63, 24, 0, 963, 964, 1, 0, 0, 0, 964, 965, 6, 88, 15, 0, 965, 966, 6, 88, 11, 0, 966, 192, 1, 0, 0, 0, 967, 968, 3, 165, 75, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 89, 13, 0, 970, 194, 1, 0, 0, 0, 971, 972, 3, 167, 76, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 90, 16, 0, 974, 196, 1, 0, 0, 0, 975, 976, 3, 337, 161, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 91, 17, 0, 978, 198, 1, 0, 0, 0, 979, 980, 3, 101, 43, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 92, 18, 0, 982, 200, 1, 0, 0, 0, 983, 984, 3, 97, 41, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 93, 19, 0, 986, 202, 1, 0, 0, 0, 987, 988, 7, 16, 0, 0, 988, 989, 7, 3, 0, 0, 989, 990, 7, 5, 0, 0, 990, 991, 7, 12, 0, 0, 991, 992, 7, 0, 0, 0, 992, 993, 7, 12, 0, 0, 993, 994, 7, 5, 0, 0, 994, 995, 7, 12, 0, 0, 995, 204, 1, 0, 0, 0, 996, 1000, 8, 32, 0, 0, 997, 998, 5, 47, 0, 0, 998, 1000, 8, 33, 0, 0, 999, 996, 1, 0, 0, 0, 999, 997, 1, 0, 0, 0, 1000, 206, 1, 0, 0, 0, 1001, 1003, 3, 205, 95, 0, 1002, 1001, 1, 0, 0, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1002, 1, 0, 0, 0, 1004, 1005, 1, 0, 0, 0, 1005, 208, 1, 0, 0, 0, 1006, 1007, 3, 207, 96, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 97, 20, 0, 1009, 210, 1, 0, 0, 0, 1010, 1011, 3, 85, 35, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 98, 21, 0, 1013, 212, 1, 0, 0, 0, 1014, 1015, 3, 57, 21, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 99, 10, 0, 1017, 214, 1, 0, 0, 0, 1018, 1019, 3, 59, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 100, 10, 0, 1021, 216, 1, 0, 0, 0, 1022, 1023, 3, 61, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 101, 10, 0, 1025, 218, 1, 0, 0, 0, 1026, 1027, 3, 63, 24, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 102, 15, 0, 1029, 1030, 6, 102, 11, 0, 1030, 220, 1, 0, 0, 0, 1031, 1032, 3, 105, 45, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1034, 6, 103, 22, 0, 1034, 222, 1, 0, 0, 0, 1035, 1036, 3, 101, 43, 0, 1036, 1037, 1, 0, 0, 0, 1037, 1038, 6, 104, 18, 0, 1038, 224, 1, 0, 0, 0, 1039, 1040, 3, 129, 57, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 105, 23, 0, 1042, 226, 1, 0, 0, 0, 1043, 1044, 3, 163, 74, 0, 1044, 1045, 1, 0, 0, 0, 1045, 1046, 6, 106, 24, 0, 1046, 228, 1, 0, 0, 0, 1047, 1052, 3, 67, 26, 0, 1048, 1052, 3, 65, 25, 0, 1049, 1052, 3, 81, 33, 0, 1050, 1052, 3, 155, 70, 0, 1051, 1047, 1, 0, 0, 0, 1051, 1048, 1, 0, 0, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1050, 1, 0, 0, 0, 1052, 230, 1, 0, 0, 0, 1053, 1056, 3, 67, 26, 0, 1054, 1056, 3, 155, 70, 0, 1055, 1053, 1, 0, 0, 0, 1055, 1054, 1, 0, 0, 0, 1056, 1060, 1, 0, 0, 0, 1057, 1059, 3, 229, 107, 0, 1058, 1057, 1, 0, 0, 0, 1059, 1062, 1, 0, 0, 0, 1060, 1058, 1, 0, 0, 0, 1060, 1061, 1, 0, 0, 0, 1061, 1073, 1, 0, 0, 0, 1062, 1060, 1, 0, 0, 0, 1063, 1066, 3, 81, 33, 0, 1064, 1066, 3, 75, 30, 0, 1065, 1063, 1, 0, 0, 0, 1065, 1064, 1, 0, 0, 0, 1066, 1068, 1, 0, 0, 0, 1067, 1069, 3, 229, 107, 0, 1068, 1067, 1, 0, 0, 0, 1069, 1070, 1, 0, 0, 0, 1070, 1068, 1, 0, 0, 0, 1070, 1071, 1, 0, 0, 0, 1071, 1073, 1, 0, 0, 0, 1072, 1055, 1, 0, 0, 0, 1072, 1065, 1, 0, 0, 0, 1073, 232, 1, 0, 0, 0, 1074, 1077, 3, 231, 108, 0, 1075, 1077, 3, 171, 78, 0, 1076, 1074, 1, 0, 0, 0, 1076, 1075, 1, 0, 0, 0, 1077, 1078, 1, 0, 0, 0, 1078, 1076, 1, 0, 0, 0, 1078, 1079, 1, 0, 0, 0, 1079, 234, 1, 0, 0, 0, 1080, 1081, 3, 57, 21, 0, 1081, 1082, 1, 0, 0, 0, 1082, 1083, 6, 110, 10, 0, 1083, 236, 1, 0, 0, 0, 1084, 1085, 3, 59, 22, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 6, 111, 10, 0, 1087, 238, 1, 0, 0, 0, 1088, 1089, 3, 61, 23, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 112, 10, 0, 1091, 240, 1, 0, 0, 0, 1092, 1093, 3, 63, 24, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1095, 6, 113, 15, 0, 1095, 1096, 6, 113, 11, 0, 1096, 242, 1, 0, 0, 0, 1097, 1098, 3, 97, 41, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 114, 19, 0, 1100, 244, 1, 0, 0, 0, 1101, 1102, 3, 101, 43, 0, 1102, 1103, 1, 0, 0, 0, 1103, 1104, 6, 115, 18, 0, 1104, 246, 1, 0, 0, 0, 1105, 1106, 3, 105, 45, 0, 1106, 1107, 1, 0, 0, 0, 1107, 1108, 6, 116, 22, 0, 1108, 248, 1, 0, 0, 0, 1109, 1110, 3, 129, 57, 0, 1110, 1111, 1, 0, 0, 0, 1111, 1112, 6, 117, 23, 0, 1112, 250, 1, 0, 0, 0, 1113, 1114, 3, 163, 74, 0, 1114, 1115, 1, 0, 0, 0, 1115, 1116, 6, 118, 24, 0, 1116, 252, 1, 0, 0, 0, 1117, 1118, 7, 12, 0, 0, 1118, 1119, 7, 2, 0, 0, 1119, 254, 1, 0, 0, 0, 1120, 1121, 3, 233, 109, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 120, 25, 0, 1123, 256, 1, 0, 0, 0, 1124, 1125, 3, 57, 21, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 121, 10, 0, 1127, 258, 1, 0, 0, 0, 1128, 1129, 3, 59, 22, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 122, 10, 0, 1131, 260, 1, 0, 0, 0, 1132, 1133, 3, 61, 23, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 123, 10, 0, 1135, 262, 1, 0, 0, 0, 1136, 1137, 3, 63, 24, 0, 1137, 1138, 1, 0, 0, 0, 1138, 1139, 6, 124, 15, 0, 1139, 1140, 6, 124, 11, 0, 1140, 264, 1, 0, 0, 0, 1141, 1142, 3, 165, 75, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 125, 13, 0, 1144, 1145, 6, 125, 26, 0, 1145, 266, 1, 0, 0, 0, 1146, 1147, 7, 7, 0, 0, 1147, 1148, 7, 9, 0, 0, 1148, 1149, 1, 0, 0, 0, 1149, 1150, 6, 126, 27, 0, 1150, 268, 1, 0, 0, 0, 1151, 1152, 7, 19, 0, 0, 1152, 1153, 7, 1, 0, 0, 1153, 1154, 7, 5, 0, 0, 1154, 1155, 7, 10, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 127, 27, 0, 1157, 270, 1, 0, 0, 0, 1158, 1159, 8, 34, 0, 0, 1159, 272, 1, 0, 0, 0, 1160, 1162, 3, 271, 128, 0, 1161, 1160, 1, 0, 0, 0, 1162, 1163, 1, 0, 0, 0, 1163, 1161, 1, 0, 0, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 1, 0, 0, 0, 1165, 1166, 3, 337, 161, 0, 1166, 1168, 1, 0, 0, 0, 1167, 1161, 1, 0, 0, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1170, 1, 0, 0, 0, 1169, 1171, 3, 271, 128, 0, 1170, 1169, 1, 0, 0, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1170, 1, 0, 0, 0, 1172, 1173, 1, 0, 0, 0, 1173, 274, 1, 0, 0, 0, 1174, 1175, 3, 273, 129, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 130, 28, 0, 1177, 276, 1, 0, 0, 0, 1178, 1179, 3, 57, 21, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 131, 10, 0, 1181, 278, 1, 0, 0, 0, 1182, 1183, 3, 59, 22, 0, 1183, 1184, 1, 0, 0, 0, 1184, 1185, 6, 132, 10, 0, 1185, 280, 1, 0, 0, 0, 1186, 1187, 3, 61, 23, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1189, 6, 133, 10, 0, 1189, 282, 1, 0, 0, 0, 1190, 1191, 3, 63, 24, 0, 1191, 1192, 1, 0, 0, 0, 1192, 1193, 6, 134, 15, 0, 1193, 1194, 6, 134, 11, 0, 1194, 1195, 6, 134, 11, 0, 1195, 284, 1, 0, 0, 0, 1196, 1197, 3, 97, 41, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 135, 19, 0, 1199, 286, 1, 0, 0, 0, 1200, 1201, 3, 101, 43, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 136, 18, 0, 1203, 288, 1, 0, 0, 0, 1204, 1205, 3, 105, 45, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 137, 22, 0, 1207, 290, 1, 0, 0, 0, 1208, 1209, 3, 269, 127, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 138, 29, 0, 1211, 292, 1, 0, 0, 0, 1212, 1213, 3, 233, 109, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 139, 25, 0, 1215, 294, 1, 0, 0, 0, 1216, 1217, 3, 173, 79, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 140, 30, 0, 1219, 296, 1, 0, 0, 0, 1220, 1221, 3, 129, 57, 0, 1221, 1222, 1, 0, 0, 0, 1222, 1223, 6, 141, 23, 0, 1223, 298, 1, 0, 0, 0, 1224, 1225, 3, 163, 74, 0, 1225, 1226, 1, 0, 0, 0, 1226, 1227, 6, 142, 24, 0, 1227, 300, 1, 0, 0, 0, 1228, 1229, 3, 57, 21, 0, 1229, 1230, 1, 0, 0, 0, 1230, 1231, 6, 143, 10, 0, 1231, 302, 1, 0, 0, 0, 1232, 1233, 3, 59, 22, 0, 1233, 1234, 1, 0, 0, 0, 1234, 1235, 6, 144, 10, 0, 1235, 304, 1, 0, 0, 0, 1236, 1237, 3, 61, 23, 0, 1237, 1238, 1, 0, 0, 0, 1238, 1239, 6, 145, 10, 0, 1239, 306, 1, 0, 0, 0, 1240, 1241, 3, 63, 24, 0, 1241, 1242, 1, 0, 0, 0, 1242, 1243, 6, 146, 15, 0, 1243, 1244, 6, 146, 11, 0, 1244, 308, 1, 0, 0, 0, 1245, 1246, 3, 105, 45, 0, 1246, 1247, 1, 0, 0, 0, 1247, 1248, 6, 147, 22, 0, 1248, 310, 1, 0, 0, 0, 1249, 1250, 3, 129, 57, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 6, 148, 23, 0, 1252, 312, 1, 0, 0, 0, 1253, 1254, 3, 163, 74, 0, 1254, 1255, 1, 0, 0, 0, 1255, 1256, 6, 149, 24, 0, 1256, 314, 1, 0, 0, 0, 1257, 1258, 3, 173, 79, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 150, 30, 0, 1260, 316, 1, 0, 0, 0, 1261, 1262, 3, 169, 77, 0, 1262, 1263, 1, 0, 0, 0, 1263, 1264, 6, 151, 31, 0, 1264, 318, 1, 0, 0, 0, 1265, 1266, 3, 57, 21, 0, 1266, 1267, 1, 0, 0, 0, 1267, 1268, 6, 152, 10, 0, 1268, 320, 1, 0, 0, 0, 1269, 1270, 3, 59, 22, 0, 1270, 1271, 1, 0, 0, 0, 1271, 1272, 6, 153, 10, 0, 1272, 322, 1, 0, 0, 0, 1273, 1274, 3, 61, 23, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 154, 10, 0, 1276, 324, 1, 0, 0, 0, 1277, 1278, 3, 63, 24, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 155, 15, 0, 1280, 1281, 6, 155, 11, 0, 1281, 326, 1, 0, 0, 0, 1282, 1283, 7, 1, 0, 0, 1283, 1284, 7, 9, 0, 0, 1284, 1285, 7, 15, 0, 0, 1285, 1286, 7, 7, 0, 0, 1286, 328, 1, 0, 0, 0, 1287, 1288, 3, 57, 21, 0, 1288, 1289, 1, 0, 0, 0, 1289, 1290, 6, 157, 10, 0, 1290, 330, 1, 0, 0, 0, 1291, 1292, 3, 59, 22, 0, 1292, 1293, 1, 0, 0, 0, 1293, 1294, 6, 158, 10, 0, 1294, 332, 1, 0, 0, 0, 1295, 1296, 3, 61, 23, 0, 1296, 1297, 1, 0, 0, 0, 1297, 1298, 6, 159, 10, 0, 1298, 334, 1, 0, 0, 0, 1299, 1300, 3, 167, 76, 0, 1300, 1301, 1, 0, 0, 0, 1301, 1302, 6, 160, 16, 0, 1302, 1303, 6, 160, 11, 0, 1303, 336, 1, 0, 0, 0, 1304, 1305, 5, 58, 0, 0, 1305, 338, 1, 0, 0, 0, 1306, 1312, 3, 75, 30, 0, 1307, 1312, 3, 65, 25, 0, 1308, 1312, 3, 105, 45, 0, 1309, 1312, 3, 67, 26, 0, 1310, 1312, 3, 81, 33, 0, 1311, 1306, 1, 0, 0, 0, 1311, 1307, 1, 0, 0, 0, 1311, 1308, 1, 0, 0, 0, 1311, 1309, 1, 0, 0, 0, 1311, 1310, 1, 0, 0, 0, 1312, 1313, 1, 0, 0, 0, 1313, 1311, 1, 0, 0, 0, 1313, 1314, 1, 0, 0, 0, 1314, 340, 1, 0, 0, 0, 1315, 1316, 3, 57, 21, 0, 1316, 1317, 1, 0, 0, 0, 1317, 1318, 6, 163, 10, 0, 1318, 342, 1, 0, 0, 0, 1319, 1320, 3, 59, 22, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 164, 10, 0, 1322, 344, 1, 0, 0, 0, 1323, 1324, 3, 61, 23, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1326, 6, 165, 10, 0, 1326, 346, 1, 0, 0, 0, 1327, 1328, 3, 63, 24, 0, 1328, 1329, 1, 0, 0, 0, 1329, 1330, 6, 166, 15, 0, 1330, 1331, 6, 166, 11, 0, 1331, 348, 1, 0, 0, 0, 1332, 1333, 3, 337, 161, 0, 1333, 1334, 1, 0, 0, 0, 1334, 1335, 6, 167, 17, 0, 1335, 350, 1, 0, 0, 0, 1336, 1337, 3, 101, 43, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 168, 18, 0, 1339, 352, 1, 0, 0, 0, 1340, 1341, 3, 105, 45, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 169, 22, 0, 1343, 354, 1, 0, 0, 0, 1344, 1345, 3, 267, 126, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 170, 32, 0, 1347, 1348, 6, 170, 33, 0, 1348, 356, 1, 0, 0, 0, 1349, 1350, 3, 207, 96, 0, 1350, 1351, 1, 0, 0, 0, 1351, 1352, 6, 171, 20, 0, 1352, 358, 1, 0, 0, 0, 1353, 1354, 3, 85, 35, 0, 1354, 1355, 1, 0, 0, 0, 1355, 1356, 6, 172, 21, 0, 1356, 360, 1, 0, 0, 0, 1357, 1358, 3, 57, 21, 0, 1358, 1359, 1, 0, 0, 0, 1359, 1360, 6, 173, 10, 0, 1360, 362, 1, 0, 0, 0, 1361, 1362, 3, 59, 22, 0, 1362, 1363, 1, 0, 0, 0, 1363, 1364, 6, 174, 10, 0, 1364, 364, 1, 0, 0, 0, 1365, 1366, 3, 61, 23, 0, 1366, 1367, 1, 0, 0, 0, 1367, 1368, 6, 175, 10, 0, 1368, 366, 1, 0, 0, 0, 1369, 1370, 3, 63, 24, 0, 1370, 1371, 1, 0, 0, 0, 1371, 1372, 6, 176, 15, 0, 1372, 1373, 6, 176, 11, 0, 1373, 1374, 6, 176, 11, 0, 1374, 368, 1, 0, 0, 0, 1375, 1376, 3, 101, 43, 0, 1376, 1377, 1, 0, 0, 0, 1377, 1378, 6, 177, 18, 0, 1378, 370, 1, 0, 0, 0, 1379, 1380, 3, 105, 45, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 178, 22, 0, 1382, 372, 1, 0, 0, 0, 1383, 1384, 3, 233, 109, 0, 1384, 1385, 1, 0, 0, 0, 1385, 1386, 6, 179, 25, 0, 1386, 374, 1, 0, 0, 0, 1387, 1388, 3, 57, 21, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 180, 10, 0, 1390, 376, 1, 0, 0, 0, 1391, 1392, 3, 59, 22, 0, 1392, 1393, 1, 0, 0, 0, 1393, 1394, 6, 181, 10, 0, 1394, 378, 1, 0, 0, 0, 1395, 1396, 3, 61, 23, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 182, 10, 0, 1398, 380, 1, 0, 0, 0, 1399, 1400, 3, 63, 24, 0, 1400, 1401, 1, 0, 0, 0, 1401, 1402, 6, 183, 15, 0, 1402, 1403, 6, 183, 11, 0, 1403, 382, 1, 0, 0, 0, 1404, 1405, 3, 207, 96, 0, 1405, 1406, 1, 0, 0, 0, 1406, 1407, 6, 184, 20, 0, 1407, 1408, 6, 184, 11, 0, 1408, 1409, 6, 184, 34, 0, 1409, 384, 1, 0, 0, 0, 1410, 1411, 3, 85, 35, 0, 1411, 1412, 1, 0, 0, 0, 1412, 1413, 6, 185, 21, 0, 1413, 1414, 6, 185, 11, 0, 1414, 1415, 6, 185, 34, 0, 1415, 386, 1, 0, 0, 0, 1416, 1417, 3, 57, 21, 0, 1417, 1418, 1, 0, 0, 0, 1418, 1419, 6, 186, 10, 0, 1419, 388, 1, 0, 0, 0, 1420, 1421, 3, 59, 22, 0, 1421, 1422, 1, 0, 0, 0, 1422, 1423, 6, 187, 10, 0, 1423, 390, 1, 0, 0, 0, 1424, 1425, 3, 61, 23, 0, 1425, 1426, 1, 0, 0, 0, 1426, 1427, 6, 188, 10, 0, 1427, 392, 1, 0, 0, 0, 1428, 1429, 3, 337, 161, 0, 1429, 1430, 1, 0, 0, 0, 1430, 1431, 6, 189, 17, 0, 1431, 1432, 6, 189, 11, 0, 1432, 1433, 6, 189, 9, 0, 1433, 394, 1, 0, 0, 0, 1434, 1435, 3, 101, 43, 0, 1435, 1436, 1, 0, 0, 0, 1436, 1437, 6, 190, 18, 0, 1437, 1438, 6, 190, 11, 0, 1438, 1439, 6, 190, 9, 0, 1439, 396, 1, 0, 0, 0, 1440, 1441, 3, 57, 21, 0, 1441, 1442, 1, 0, 0, 0, 1442, 1443, 6, 191, 10, 0, 1443, 398, 1, 0, 0, 0, 1444, 1445, 3, 59, 22, 0, 1445, 1446, 1, 0, 0, 0, 1446, 1447, 6, 192, 10, 0, 1447, 400, 1, 0, 0, 0, 1448, 1449, 3, 61, 23, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 193, 10, 0, 1451, 402, 1, 0, 0, 0, 1452, 1453, 3, 173, 79, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 194, 11, 0, 1455, 1456, 6, 194, 0, 0, 1456, 1457, 6, 194, 30, 0, 1457, 404, 1, 0, 0, 0, 1458, 1459, 3, 169, 77, 0, 1459, 1460, 1, 0, 0, 0, 1460, 1461, 6, 195, 11, 0, 1461, 1462, 6, 195, 0, 0, 1462, 1463, 6, 195, 31, 0, 1463, 406, 1, 0, 0, 0, 1464, 1465, 3, 91, 38, 0, 1465, 1466, 1, 0, 0, 0, 1466, 1467, 6, 196, 11, 0, 1467, 1468, 6, 196, 0, 0, 1468, 1469, 6, 196, 35, 0, 1469, 408, 1, 0, 0, 0, 1470, 1471, 3, 63, 24, 0, 1471, 1472, 1, 0, 0, 0, 1472, 1473, 6, 197, 15, 0, 1473, 1474, 6, 197, 11, 0, 1474, 410, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 588, 598, 602, 605, 614, 616, 627, 646, 651, 660, 667, 672, 674, 685, 693, 696, 698, 703, 708, 714, 721, 726, 732, 735, 743, 747, 874, 879, 886, 888, 904, 909, 914, 916, 922, 999, 1004, 1051, 1055, 1060, 1065, 1070, 1072, 1076, 1078, 1163, 1167, 1172, 1311, 1313, 36, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 19, 0, 7, 65, 0, 5, 0, 0, 7, 25, 0, 7, 66, 0, 7, 104, 0, 7, 34, 0, 7, 32, 0, 7, 76, 0, 7, 26, 0, 7, 36, 0, 7, 48, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 29, 0] \ No newline at end of file +[4, 0, 120, 1472, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 4, 19, 578, 8, 19, 11, 19, 12, 19, 579, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 588, 8, 20, 10, 20, 12, 20, 591, 9, 20, 1, 20, 3, 20, 594, 8, 20, 1, 20, 3, 20, 597, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 606, 8, 21, 10, 21, 12, 21, 609, 9, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 4, 22, 617, 8, 22, 11, 22, 12, 22, 618, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 3, 28, 638, 8, 28, 1, 28, 4, 28, 641, 8, 28, 11, 28, 12, 28, 642, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 652, 8, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 659, 8, 33, 1, 34, 1, 34, 1, 34, 5, 34, 664, 8, 34, 10, 34, 12, 34, 667, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 675, 8, 34, 10, 34, 12, 34, 678, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 685, 8, 34, 1, 34, 3, 34, 688, 8, 34, 3, 34, 690, 8, 34, 1, 35, 4, 35, 693, 8, 35, 11, 35, 12, 35, 694, 1, 36, 4, 36, 698, 8, 36, 11, 36, 12, 36, 699, 1, 36, 1, 36, 5, 36, 704, 8, 36, 10, 36, 12, 36, 707, 9, 36, 1, 36, 1, 36, 4, 36, 711, 8, 36, 11, 36, 12, 36, 712, 1, 36, 4, 36, 716, 8, 36, 11, 36, 12, 36, 717, 1, 36, 1, 36, 5, 36, 722, 8, 36, 10, 36, 12, 36, 725, 9, 36, 3, 36, 727, 8, 36, 1, 36, 1, 36, 1, 36, 1, 36, 4, 36, 733, 8, 36, 11, 36, 12, 36, 734, 1, 36, 1, 36, 3, 36, 739, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 872, 8, 74, 1, 74, 5, 74, 875, 8, 74, 10, 74, 12, 74, 878, 9, 74, 1, 74, 1, 74, 4, 74, 882, 8, 74, 11, 74, 12, 74, 883, 3, 74, 886, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 900, 8, 77, 10, 77, 12, 77, 903, 9, 77, 1, 77, 1, 77, 3, 77, 907, 8, 77, 1, 77, 4, 77, 910, 8, 77, 11, 77, 12, 77, 911, 3, 77, 914, 8, 77, 1, 78, 1, 78, 4, 78, 918, 8, 78, 11, 78, 12, 78, 919, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 997, 8, 95, 1, 96, 4, 96, 1000, 8, 96, 11, 96, 12, 96, 1001, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 3, 107, 1049, 8, 107, 1, 108, 1, 108, 3, 108, 1053, 8, 108, 1, 108, 5, 108, 1056, 8, 108, 10, 108, 12, 108, 1059, 9, 108, 1, 108, 1, 108, 3, 108, 1063, 8, 108, 1, 108, 4, 108, 1066, 8, 108, 11, 108, 12, 108, 1067, 3, 108, 1070, 8, 108, 1, 109, 1, 109, 4, 109, 1074, 8, 109, 11, 109, 12, 109, 1075, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 129, 4, 129, 1159, 8, 129, 11, 129, 12, 129, 1160, 1, 129, 1, 129, 3, 129, 1165, 8, 129, 1, 129, 4, 129, 1168, 8, 129, 11, 129, 12, 129, 1169, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 4, 162, 1309, 8, 162, 11, 162, 12, 162, 1310, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 2, 607, 676, 0, 198, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 0, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 25, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 0, 233, 80, 235, 81, 237, 82, 239, 83, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 84, 255, 0, 257, 85, 259, 86, 261, 87, 263, 0, 265, 0, 267, 88, 269, 89, 271, 0, 273, 90, 275, 0, 277, 91, 279, 92, 281, 93, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 0, 301, 94, 303, 95, 305, 96, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 97, 321, 98, 323, 99, 325, 0, 327, 100, 329, 101, 331, 102, 333, 103, 335, 0, 337, 104, 339, 105, 341, 106, 343, 107, 345, 108, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 0, 361, 109, 363, 110, 365, 111, 367, 0, 369, 0, 371, 0, 373, 0, 375, 112, 377, 113, 379, 114, 381, 0, 383, 0, 385, 0, 387, 115, 389, 116, 391, 117, 393, 0, 395, 0, 397, 118, 399, 119, 401, 120, 403, 0, 405, 0, 407, 0, 409, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1500, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 1, 83, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 227, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 9, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 10, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 12, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 13, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 14, 409, 1, 0, 0, 0, 15, 411, 1, 0, 0, 0, 17, 421, 1, 0, 0, 0, 19, 428, 1, 0, 0, 0, 21, 437, 1, 0, 0, 0, 23, 444, 1, 0, 0, 0, 25, 454, 1, 0, 0, 0, 27, 461, 1, 0, 0, 0, 29, 468, 1, 0, 0, 0, 31, 475, 1, 0, 0, 0, 33, 483, 1, 0, 0, 0, 35, 495, 1, 0, 0, 0, 37, 504, 1, 0, 0, 0, 39, 510, 1, 0, 0, 0, 41, 517, 1, 0, 0, 0, 43, 524, 1, 0, 0, 0, 45, 532, 1, 0, 0, 0, 47, 540, 1, 0, 0, 0, 49, 555, 1, 0, 0, 0, 51, 565, 1, 0, 0, 0, 53, 577, 1, 0, 0, 0, 55, 583, 1, 0, 0, 0, 57, 600, 1, 0, 0, 0, 59, 616, 1, 0, 0, 0, 61, 622, 1, 0, 0, 0, 63, 626, 1, 0, 0, 0, 65, 628, 1, 0, 0, 0, 67, 630, 1, 0, 0, 0, 69, 633, 1, 0, 0, 0, 71, 635, 1, 0, 0, 0, 73, 644, 1, 0, 0, 0, 75, 646, 1, 0, 0, 0, 77, 651, 1, 0, 0, 0, 79, 653, 1, 0, 0, 0, 81, 658, 1, 0, 0, 0, 83, 689, 1, 0, 0, 0, 85, 692, 1, 0, 0, 0, 87, 738, 1, 0, 0, 0, 89, 740, 1, 0, 0, 0, 91, 743, 1, 0, 0, 0, 93, 747, 1, 0, 0, 0, 95, 751, 1, 0, 0, 0, 97, 753, 1, 0, 0, 0, 99, 756, 1, 0, 0, 0, 101, 758, 1, 0, 0, 0, 103, 763, 1, 0, 0, 0, 105, 765, 1, 0, 0, 0, 107, 771, 1, 0, 0, 0, 109, 777, 1, 0, 0, 0, 111, 780, 1, 0, 0, 0, 113, 783, 1, 0, 0, 0, 115, 788, 1, 0, 0, 0, 117, 793, 1, 0, 0, 0, 119, 795, 1, 0, 0, 0, 121, 799, 1, 0, 0, 0, 123, 804, 1, 0, 0, 0, 125, 810, 1, 0, 0, 0, 127, 813, 1, 0, 0, 0, 129, 815, 1, 0, 0, 0, 131, 821, 1, 0, 0, 0, 133, 823, 1, 0, 0, 0, 135, 828, 1, 0, 0, 0, 137, 831, 1, 0, 0, 0, 139, 834, 1, 0, 0, 0, 141, 837, 1, 0, 0, 0, 143, 839, 1, 0, 0, 0, 145, 842, 1, 0, 0, 0, 147, 844, 1, 0, 0, 0, 149, 847, 1, 0, 0, 0, 151, 849, 1, 0, 0, 0, 153, 851, 1, 0, 0, 0, 155, 853, 1, 0, 0, 0, 157, 855, 1, 0, 0, 0, 159, 857, 1, 0, 0, 0, 161, 863, 1, 0, 0, 0, 163, 885, 1, 0, 0, 0, 165, 887, 1, 0, 0, 0, 167, 892, 1, 0, 0, 0, 169, 913, 1, 0, 0, 0, 171, 915, 1, 0, 0, 0, 173, 923, 1, 0, 0, 0, 175, 925, 1, 0, 0, 0, 177, 929, 1, 0, 0, 0, 179, 933, 1, 0, 0, 0, 181, 937, 1, 0, 0, 0, 183, 942, 1, 0, 0, 0, 185, 947, 1, 0, 0, 0, 187, 951, 1, 0, 0, 0, 189, 955, 1, 0, 0, 0, 191, 959, 1, 0, 0, 0, 193, 964, 1, 0, 0, 0, 195, 968, 1, 0, 0, 0, 197, 972, 1, 0, 0, 0, 199, 976, 1, 0, 0, 0, 201, 980, 1, 0, 0, 0, 203, 984, 1, 0, 0, 0, 205, 996, 1, 0, 0, 0, 207, 999, 1, 0, 0, 0, 209, 1003, 1, 0, 0, 0, 211, 1007, 1, 0, 0, 0, 213, 1011, 1, 0, 0, 0, 215, 1015, 1, 0, 0, 0, 217, 1019, 1, 0, 0, 0, 219, 1023, 1, 0, 0, 0, 221, 1028, 1, 0, 0, 0, 223, 1032, 1, 0, 0, 0, 225, 1036, 1, 0, 0, 0, 227, 1040, 1, 0, 0, 0, 229, 1048, 1, 0, 0, 0, 231, 1069, 1, 0, 0, 0, 233, 1073, 1, 0, 0, 0, 235, 1077, 1, 0, 0, 0, 237, 1081, 1, 0, 0, 0, 239, 1085, 1, 0, 0, 0, 241, 1089, 1, 0, 0, 0, 243, 1094, 1, 0, 0, 0, 245, 1098, 1, 0, 0, 0, 247, 1102, 1, 0, 0, 0, 249, 1106, 1, 0, 0, 0, 251, 1110, 1, 0, 0, 0, 253, 1114, 1, 0, 0, 0, 255, 1117, 1, 0, 0, 0, 257, 1121, 1, 0, 0, 0, 259, 1125, 1, 0, 0, 0, 261, 1129, 1, 0, 0, 0, 263, 1133, 1, 0, 0, 0, 265, 1138, 1, 0, 0, 0, 267, 1143, 1, 0, 0, 0, 269, 1148, 1, 0, 0, 0, 271, 1155, 1, 0, 0, 0, 273, 1164, 1, 0, 0, 0, 275, 1171, 1, 0, 0, 0, 277, 1175, 1, 0, 0, 0, 279, 1179, 1, 0, 0, 0, 281, 1183, 1, 0, 0, 0, 283, 1187, 1, 0, 0, 0, 285, 1193, 1, 0, 0, 0, 287, 1197, 1, 0, 0, 0, 289, 1201, 1, 0, 0, 0, 291, 1205, 1, 0, 0, 0, 293, 1209, 1, 0, 0, 0, 295, 1213, 1, 0, 0, 0, 297, 1217, 1, 0, 0, 0, 299, 1221, 1, 0, 0, 0, 301, 1225, 1, 0, 0, 0, 303, 1229, 1, 0, 0, 0, 305, 1233, 1, 0, 0, 0, 307, 1237, 1, 0, 0, 0, 309, 1242, 1, 0, 0, 0, 311, 1246, 1, 0, 0, 0, 313, 1250, 1, 0, 0, 0, 315, 1254, 1, 0, 0, 0, 317, 1258, 1, 0, 0, 0, 319, 1262, 1, 0, 0, 0, 321, 1266, 1, 0, 0, 0, 323, 1270, 1, 0, 0, 0, 325, 1274, 1, 0, 0, 0, 327, 1279, 1, 0, 0, 0, 329, 1284, 1, 0, 0, 0, 331, 1288, 1, 0, 0, 0, 333, 1292, 1, 0, 0, 0, 335, 1296, 1, 0, 0, 0, 337, 1301, 1, 0, 0, 0, 339, 1308, 1, 0, 0, 0, 341, 1312, 1, 0, 0, 0, 343, 1316, 1, 0, 0, 0, 345, 1320, 1, 0, 0, 0, 347, 1324, 1, 0, 0, 0, 349, 1329, 1, 0, 0, 0, 351, 1333, 1, 0, 0, 0, 353, 1337, 1, 0, 0, 0, 355, 1341, 1, 0, 0, 0, 357, 1346, 1, 0, 0, 0, 359, 1350, 1, 0, 0, 0, 361, 1354, 1, 0, 0, 0, 363, 1358, 1, 0, 0, 0, 365, 1362, 1, 0, 0, 0, 367, 1366, 1, 0, 0, 0, 369, 1372, 1, 0, 0, 0, 371, 1376, 1, 0, 0, 0, 373, 1380, 1, 0, 0, 0, 375, 1384, 1, 0, 0, 0, 377, 1388, 1, 0, 0, 0, 379, 1392, 1, 0, 0, 0, 381, 1396, 1, 0, 0, 0, 383, 1401, 1, 0, 0, 0, 385, 1407, 1, 0, 0, 0, 387, 1413, 1, 0, 0, 0, 389, 1417, 1, 0, 0, 0, 391, 1421, 1, 0, 0, 0, 393, 1425, 1, 0, 0, 0, 395, 1431, 1, 0, 0, 0, 397, 1437, 1, 0, 0, 0, 399, 1441, 1, 0, 0, 0, 401, 1445, 1, 0, 0, 0, 403, 1449, 1, 0, 0, 0, 405, 1455, 1, 0, 0, 0, 407, 1461, 1, 0, 0, 0, 409, 1467, 1, 0, 0, 0, 411, 412, 7, 0, 0, 0, 412, 413, 7, 1, 0, 0, 413, 414, 7, 2, 0, 0, 414, 415, 7, 2, 0, 0, 415, 416, 7, 3, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 5, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 0, 0, 0, 420, 16, 1, 0, 0, 0, 421, 422, 7, 0, 0, 0, 422, 423, 7, 6, 0, 0, 423, 424, 7, 7, 0, 0, 424, 425, 7, 8, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 1, 1, 0, 427, 18, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 9, 0, 0, 430, 431, 7, 6, 0, 0, 431, 432, 7, 1, 0, 0, 432, 433, 7, 4, 0, 0, 433, 434, 7, 10, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 2, 2, 0, 436, 20, 1, 0, 0, 0, 437, 438, 7, 3, 0, 0, 438, 439, 7, 11, 0, 0, 439, 440, 7, 12, 0, 0, 440, 441, 7, 13, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 3, 0, 0, 443, 22, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 446, 7, 14, 0, 0, 446, 447, 7, 8, 0, 0, 447, 448, 7, 13, 0, 0, 448, 449, 7, 12, 0, 0, 449, 450, 7, 1, 0, 0, 450, 451, 7, 9, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 4, 3, 0, 453, 24, 1, 0, 0, 0, 454, 455, 7, 15, 0, 0, 455, 456, 7, 6, 0, 0, 456, 457, 7, 7, 0, 0, 457, 458, 7, 16, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 5, 4, 0, 460, 26, 1, 0, 0, 0, 461, 462, 7, 17, 0, 0, 462, 463, 7, 6, 0, 0, 463, 464, 7, 7, 0, 0, 464, 465, 7, 18, 0, 0, 465, 466, 1, 0, 0, 0, 466, 467, 6, 6, 0, 0, 467, 28, 1, 0, 0, 0, 468, 469, 7, 18, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 8, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 1, 0, 474, 30, 1, 0, 0, 0, 475, 476, 7, 13, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 16, 0, 0, 478, 479, 7, 1, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 8, 0, 0, 482, 32, 1, 0, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 11, 0, 0, 485, 486, 5, 95, 0, 0, 486, 487, 7, 3, 0, 0, 487, 488, 7, 14, 0, 0, 488, 489, 7, 8, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 9, 0, 0, 491, 492, 7, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 9, 5, 0, 494, 34, 1, 0, 0, 0, 495, 496, 7, 6, 0, 0, 496, 497, 7, 3, 0, 0, 497, 498, 7, 9, 0, 0, 498, 499, 7, 12, 0, 0, 499, 500, 7, 16, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 10, 6, 0, 503, 36, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 506, 7, 7, 0, 0, 506, 507, 7, 19, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 11, 0, 0, 509, 38, 1, 0, 0, 0, 510, 511, 7, 2, 0, 0, 511, 512, 7, 10, 0, 0, 512, 513, 7, 7, 0, 0, 513, 514, 7, 19, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 12, 7, 0, 516, 40, 1, 0, 0, 0, 517, 518, 7, 2, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 13, 0, 0, 523, 42, 1, 0, 0, 0, 524, 525, 7, 2, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 12, 0, 0, 527, 528, 7, 5, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 14, 0, 0, 531, 44, 1, 0, 0, 0, 532, 533, 7, 19, 0, 0, 533, 534, 7, 10, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 7, 6, 0, 0, 536, 537, 7, 3, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 15, 0, 0, 539, 46, 1, 0, 0, 0, 540, 541, 4, 16, 0, 0, 541, 542, 7, 1, 0, 0, 542, 543, 7, 9, 0, 0, 543, 544, 7, 13, 0, 0, 544, 545, 7, 1, 0, 0, 545, 546, 7, 9, 0, 0, 546, 547, 7, 3, 0, 0, 547, 548, 7, 2, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 12, 0, 0, 550, 551, 7, 5, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 16, 0, 0, 554, 48, 1, 0, 0, 0, 555, 556, 4, 17, 1, 0, 556, 557, 7, 13, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 7, 0, 0, 559, 560, 7, 18, 0, 0, 560, 561, 7, 20, 0, 0, 561, 562, 7, 8, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 17, 8, 0, 564, 50, 1, 0, 0, 0, 565, 566, 4, 18, 2, 0, 566, 567, 7, 16, 0, 0, 567, 568, 7, 3, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 6, 0, 0, 570, 571, 7, 1, 0, 0, 571, 572, 7, 4, 0, 0, 572, 573, 7, 2, 0, 0, 573, 574, 1, 0, 0, 0, 574, 575, 6, 18, 9, 0, 575, 52, 1, 0, 0, 0, 576, 578, 8, 21, 0, 0, 577, 576, 1, 0, 0, 0, 578, 579, 1, 0, 0, 0, 579, 577, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 582, 6, 19, 0, 0, 582, 54, 1, 0, 0, 0, 583, 584, 5, 47, 0, 0, 584, 585, 5, 47, 0, 0, 585, 589, 1, 0, 0, 0, 586, 588, 8, 22, 0, 0, 587, 586, 1, 0, 0, 0, 588, 591, 1, 0, 0, 0, 589, 587, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 593, 1, 0, 0, 0, 591, 589, 1, 0, 0, 0, 592, 594, 5, 13, 0, 0, 593, 592, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 596, 1, 0, 0, 0, 595, 597, 5, 10, 0, 0, 596, 595, 1, 0, 0, 0, 596, 597, 1, 0, 0, 0, 597, 598, 1, 0, 0, 0, 598, 599, 6, 20, 10, 0, 599, 56, 1, 0, 0, 0, 600, 601, 5, 47, 0, 0, 601, 602, 5, 42, 0, 0, 602, 607, 1, 0, 0, 0, 603, 606, 3, 57, 21, 0, 604, 606, 9, 0, 0, 0, 605, 603, 1, 0, 0, 0, 605, 604, 1, 0, 0, 0, 606, 609, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 608, 610, 1, 0, 0, 0, 609, 607, 1, 0, 0, 0, 610, 611, 5, 42, 0, 0, 611, 612, 5, 47, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 6, 21, 10, 0, 614, 58, 1, 0, 0, 0, 615, 617, 7, 23, 0, 0, 616, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 620, 1, 0, 0, 0, 620, 621, 6, 22, 10, 0, 621, 60, 1, 0, 0, 0, 622, 623, 5, 124, 0, 0, 623, 624, 1, 0, 0, 0, 624, 625, 6, 23, 11, 0, 625, 62, 1, 0, 0, 0, 626, 627, 7, 24, 0, 0, 627, 64, 1, 0, 0, 0, 628, 629, 7, 25, 0, 0, 629, 66, 1, 0, 0, 0, 630, 631, 5, 92, 0, 0, 631, 632, 7, 26, 0, 0, 632, 68, 1, 0, 0, 0, 633, 634, 8, 27, 0, 0, 634, 70, 1, 0, 0, 0, 635, 637, 7, 3, 0, 0, 636, 638, 7, 28, 0, 0, 637, 636, 1, 0, 0, 0, 637, 638, 1, 0, 0, 0, 638, 640, 1, 0, 0, 0, 639, 641, 3, 63, 24, 0, 640, 639, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 640, 1, 0, 0, 0, 642, 643, 1, 0, 0, 0, 643, 72, 1, 0, 0, 0, 644, 645, 5, 64, 0, 0, 645, 74, 1, 0, 0, 0, 646, 647, 5, 96, 0, 0, 647, 76, 1, 0, 0, 0, 648, 652, 8, 29, 0, 0, 649, 650, 5, 96, 0, 0, 650, 652, 5, 96, 0, 0, 651, 648, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 652, 78, 1, 0, 0, 0, 653, 654, 5, 95, 0, 0, 654, 80, 1, 0, 0, 0, 655, 659, 3, 65, 25, 0, 656, 659, 3, 63, 24, 0, 657, 659, 3, 79, 32, 0, 658, 655, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 658, 657, 1, 0, 0, 0, 659, 82, 1, 0, 0, 0, 660, 665, 5, 34, 0, 0, 661, 664, 3, 67, 26, 0, 662, 664, 3, 69, 27, 0, 663, 661, 1, 0, 0, 0, 663, 662, 1, 0, 0, 0, 664, 667, 1, 0, 0, 0, 665, 663, 1, 0, 0, 0, 665, 666, 1, 0, 0, 0, 666, 668, 1, 0, 0, 0, 667, 665, 1, 0, 0, 0, 668, 690, 5, 34, 0, 0, 669, 670, 5, 34, 0, 0, 670, 671, 5, 34, 0, 0, 671, 672, 5, 34, 0, 0, 672, 676, 1, 0, 0, 0, 673, 675, 8, 22, 0, 0, 674, 673, 1, 0, 0, 0, 675, 678, 1, 0, 0, 0, 676, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 679, 1, 0, 0, 0, 678, 676, 1, 0, 0, 0, 679, 680, 5, 34, 0, 0, 680, 681, 5, 34, 0, 0, 681, 682, 5, 34, 0, 0, 682, 684, 1, 0, 0, 0, 683, 685, 5, 34, 0, 0, 684, 683, 1, 0, 0, 0, 684, 685, 1, 0, 0, 0, 685, 687, 1, 0, 0, 0, 686, 688, 5, 34, 0, 0, 687, 686, 1, 0, 0, 0, 687, 688, 1, 0, 0, 0, 688, 690, 1, 0, 0, 0, 689, 660, 1, 0, 0, 0, 689, 669, 1, 0, 0, 0, 690, 84, 1, 0, 0, 0, 691, 693, 3, 63, 24, 0, 692, 691, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 692, 1, 0, 0, 0, 694, 695, 1, 0, 0, 0, 695, 86, 1, 0, 0, 0, 696, 698, 3, 63, 24, 0, 697, 696, 1, 0, 0, 0, 698, 699, 1, 0, 0, 0, 699, 697, 1, 0, 0, 0, 699, 700, 1, 0, 0, 0, 700, 701, 1, 0, 0, 0, 701, 705, 3, 103, 44, 0, 702, 704, 3, 63, 24, 0, 703, 702, 1, 0, 0, 0, 704, 707, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 705, 706, 1, 0, 0, 0, 706, 739, 1, 0, 0, 0, 707, 705, 1, 0, 0, 0, 708, 710, 3, 103, 44, 0, 709, 711, 3, 63, 24, 0, 710, 709, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 710, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 739, 1, 0, 0, 0, 714, 716, 3, 63, 24, 0, 715, 714, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 715, 1, 0, 0, 0, 717, 718, 1, 0, 0, 0, 718, 726, 1, 0, 0, 0, 719, 723, 3, 103, 44, 0, 720, 722, 3, 63, 24, 0, 721, 720, 1, 0, 0, 0, 722, 725, 1, 0, 0, 0, 723, 721, 1, 0, 0, 0, 723, 724, 1, 0, 0, 0, 724, 727, 1, 0, 0, 0, 725, 723, 1, 0, 0, 0, 726, 719, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0, 728, 729, 3, 71, 28, 0, 729, 739, 1, 0, 0, 0, 730, 732, 3, 103, 44, 0, 731, 733, 3, 63, 24, 0, 732, 731, 1, 0, 0, 0, 733, 734, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 734, 735, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 3, 71, 28, 0, 737, 739, 1, 0, 0, 0, 738, 697, 1, 0, 0, 0, 738, 708, 1, 0, 0, 0, 738, 715, 1, 0, 0, 0, 738, 730, 1, 0, 0, 0, 739, 88, 1, 0, 0, 0, 740, 741, 7, 30, 0, 0, 741, 742, 7, 31, 0, 0, 742, 90, 1, 0, 0, 0, 743, 744, 7, 12, 0, 0, 744, 745, 7, 9, 0, 0, 745, 746, 7, 0, 0, 0, 746, 92, 1, 0, 0, 0, 747, 748, 7, 12, 0, 0, 748, 749, 7, 2, 0, 0, 749, 750, 7, 4, 0, 0, 750, 94, 1, 0, 0, 0, 751, 752, 5, 61, 0, 0, 752, 96, 1, 0, 0, 0, 753, 754, 5, 58, 0, 0, 754, 755, 5, 58, 0, 0, 755, 98, 1, 0, 0, 0, 756, 757, 5, 44, 0, 0, 757, 100, 1, 0, 0, 0, 758, 759, 7, 0, 0, 0, 759, 760, 7, 3, 0, 0, 760, 761, 7, 2, 0, 0, 761, 762, 7, 4, 0, 0, 762, 102, 1, 0, 0, 0, 763, 764, 5, 46, 0, 0, 764, 104, 1, 0, 0, 0, 765, 766, 7, 15, 0, 0, 766, 767, 7, 12, 0, 0, 767, 768, 7, 13, 0, 0, 768, 769, 7, 2, 0, 0, 769, 770, 7, 3, 0, 0, 770, 106, 1, 0, 0, 0, 771, 772, 7, 15, 0, 0, 772, 773, 7, 1, 0, 0, 773, 774, 7, 6, 0, 0, 774, 775, 7, 2, 0, 0, 775, 776, 7, 5, 0, 0, 776, 108, 1, 0, 0, 0, 777, 778, 7, 1, 0, 0, 778, 779, 7, 9, 0, 0, 779, 110, 1, 0, 0, 0, 780, 781, 7, 1, 0, 0, 781, 782, 7, 2, 0, 0, 782, 112, 1, 0, 0, 0, 783, 784, 7, 13, 0, 0, 784, 785, 7, 12, 0, 0, 785, 786, 7, 2, 0, 0, 786, 787, 7, 5, 0, 0, 787, 114, 1, 0, 0, 0, 788, 789, 7, 13, 0, 0, 789, 790, 7, 1, 0, 0, 790, 791, 7, 18, 0, 0, 791, 792, 7, 3, 0, 0, 792, 116, 1, 0, 0, 0, 793, 794, 5, 40, 0, 0, 794, 118, 1, 0, 0, 0, 795, 796, 7, 9, 0, 0, 796, 797, 7, 7, 0, 0, 797, 798, 7, 5, 0, 0, 798, 120, 1, 0, 0, 0, 799, 800, 7, 9, 0, 0, 800, 801, 7, 20, 0, 0, 801, 802, 7, 13, 0, 0, 802, 803, 7, 13, 0, 0, 803, 122, 1, 0, 0, 0, 804, 805, 7, 9, 0, 0, 805, 806, 7, 20, 0, 0, 806, 807, 7, 13, 0, 0, 807, 808, 7, 13, 0, 0, 808, 809, 7, 2, 0, 0, 809, 124, 1, 0, 0, 0, 810, 811, 7, 7, 0, 0, 811, 812, 7, 6, 0, 0, 812, 126, 1, 0, 0, 0, 813, 814, 5, 63, 0, 0, 814, 128, 1, 0, 0, 0, 815, 816, 7, 6, 0, 0, 816, 817, 7, 13, 0, 0, 817, 818, 7, 1, 0, 0, 818, 819, 7, 18, 0, 0, 819, 820, 7, 3, 0, 0, 820, 130, 1, 0, 0, 0, 821, 822, 5, 41, 0, 0, 822, 132, 1, 0, 0, 0, 823, 824, 7, 5, 0, 0, 824, 825, 7, 6, 0, 0, 825, 826, 7, 20, 0, 0, 826, 827, 7, 3, 0, 0, 827, 134, 1, 0, 0, 0, 828, 829, 5, 61, 0, 0, 829, 830, 5, 61, 0, 0, 830, 136, 1, 0, 0, 0, 831, 832, 5, 61, 0, 0, 832, 833, 5, 126, 0, 0, 833, 138, 1, 0, 0, 0, 834, 835, 5, 33, 0, 0, 835, 836, 5, 61, 0, 0, 836, 140, 1, 0, 0, 0, 837, 838, 5, 60, 0, 0, 838, 142, 1, 0, 0, 0, 839, 840, 5, 60, 0, 0, 840, 841, 5, 61, 0, 0, 841, 144, 1, 0, 0, 0, 842, 843, 5, 62, 0, 0, 843, 146, 1, 0, 0, 0, 844, 845, 5, 62, 0, 0, 845, 846, 5, 61, 0, 0, 846, 148, 1, 0, 0, 0, 847, 848, 5, 43, 0, 0, 848, 150, 1, 0, 0, 0, 849, 850, 5, 45, 0, 0, 850, 152, 1, 0, 0, 0, 851, 852, 5, 42, 0, 0, 852, 154, 1, 0, 0, 0, 853, 854, 5, 47, 0, 0, 854, 156, 1, 0, 0, 0, 855, 856, 5, 37, 0, 0, 856, 158, 1, 0, 0, 0, 857, 858, 7, 16, 0, 0, 858, 859, 7, 12, 0, 0, 859, 860, 7, 5, 0, 0, 860, 861, 7, 4, 0, 0, 861, 862, 7, 10, 0, 0, 862, 160, 1, 0, 0, 0, 863, 864, 4, 73, 3, 0, 864, 865, 3, 45, 15, 0, 865, 866, 1, 0, 0, 0, 866, 867, 6, 73, 12, 0, 867, 162, 1, 0, 0, 0, 868, 871, 3, 127, 56, 0, 869, 872, 3, 65, 25, 0, 870, 872, 3, 79, 32, 0, 871, 869, 1, 0, 0, 0, 871, 870, 1, 0, 0, 0, 872, 876, 1, 0, 0, 0, 873, 875, 3, 81, 33, 0, 874, 873, 1, 0, 0, 0, 875, 878, 1, 0, 0, 0, 876, 874, 1, 0, 0, 0, 876, 877, 1, 0, 0, 0, 877, 886, 1, 0, 0, 0, 878, 876, 1, 0, 0, 0, 879, 881, 3, 127, 56, 0, 880, 882, 3, 63, 24, 0, 881, 880, 1, 0, 0, 0, 882, 883, 1, 0, 0, 0, 883, 881, 1, 0, 0, 0, 883, 884, 1, 0, 0, 0, 884, 886, 1, 0, 0, 0, 885, 868, 1, 0, 0, 0, 885, 879, 1, 0, 0, 0, 886, 164, 1, 0, 0, 0, 887, 888, 5, 91, 0, 0, 888, 889, 1, 0, 0, 0, 889, 890, 6, 75, 0, 0, 890, 891, 6, 75, 0, 0, 891, 166, 1, 0, 0, 0, 892, 893, 5, 93, 0, 0, 893, 894, 1, 0, 0, 0, 894, 895, 6, 76, 11, 0, 895, 896, 6, 76, 11, 0, 896, 168, 1, 0, 0, 0, 897, 901, 3, 65, 25, 0, 898, 900, 3, 81, 33, 0, 899, 898, 1, 0, 0, 0, 900, 903, 1, 0, 0, 0, 901, 899, 1, 0, 0, 0, 901, 902, 1, 0, 0, 0, 902, 914, 1, 0, 0, 0, 903, 901, 1, 0, 0, 0, 904, 907, 3, 79, 32, 0, 905, 907, 3, 73, 29, 0, 906, 904, 1, 0, 0, 0, 906, 905, 1, 0, 0, 0, 907, 909, 1, 0, 0, 0, 908, 910, 3, 81, 33, 0, 909, 908, 1, 0, 0, 0, 910, 911, 1, 0, 0, 0, 911, 909, 1, 0, 0, 0, 911, 912, 1, 0, 0, 0, 912, 914, 1, 0, 0, 0, 913, 897, 1, 0, 0, 0, 913, 906, 1, 0, 0, 0, 914, 170, 1, 0, 0, 0, 915, 917, 3, 75, 30, 0, 916, 918, 3, 77, 31, 0, 917, 916, 1, 0, 0, 0, 918, 919, 1, 0, 0, 0, 919, 917, 1, 0, 0, 0, 919, 920, 1, 0, 0, 0, 920, 921, 1, 0, 0, 0, 921, 922, 3, 75, 30, 0, 922, 172, 1, 0, 0, 0, 923, 924, 3, 171, 78, 0, 924, 174, 1, 0, 0, 0, 925, 926, 3, 55, 20, 0, 926, 927, 1, 0, 0, 0, 927, 928, 6, 80, 10, 0, 928, 176, 1, 0, 0, 0, 929, 930, 3, 57, 21, 0, 930, 931, 1, 0, 0, 0, 931, 932, 6, 81, 10, 0, 932, 178, 1, 0, 0, 0, 933, 934, 3, 59, 22, 0, 934, 935, 1, 0, 0, 0, 935, 936, 6, 82, 10, 0, 936, 180, 1, 0, 0, 0, 937, 938, 3, 165, 75, 0, 938, 939, 1, 0, 0, 0, 939, 940, 6, 83, 13, 0, 940, 941, 6, 83, 14, 0, 941, 182, 1, 0, 0, 0, 942, 943, 3, 61, 23, 0, 943, 944, 1, 0, 0, 0, 944, 945, 6, 84, 15, 0, 945, 946, 6, 84, 11, 0, 946, 184, 1, 0, 0, 0, 947, 948, 3, 59, 22, 0, 948, 949, 1, 0, 0, 0, 949, 950, 6, 85, 10, 0, 950, 186, 1, 0, 0, 0, 951, 952, 3, 55, 20, 0, 952, 953, 1, 0, 0, 0, 953, 954, 6, 86, 10, 0, 954, 188, 1, 0, 0, 0, 955, 956, 3, 57, 21, 0, 956, 957, 1, 0, 0, 0, 957, 958, 6, 87, 10, 0, 958, 190, 1, 0, 0, 0, 959, 960, 3, 61, 23, 0, 960, 961, 1, 0, 0, 0, 961, 962, 6, 88, 15, 0, 962, 963, 6, 88, 11, 0, 963, 192, 1, 0, 0, 0, 964, 965, 3, 165, 75, 0, 965, 966, 1, 0, 0, 0, 966, 967, 6, 89, 13, 0, 967, 194, 1, 0, 0, 0, 968, 969, 3, 167, 76, 0, 969, 970, 1, 0, 0, 0, 970, 971, 6, 90, 16, 0, 971, 196, 1, 0, 0, 0, 972, 973, 3, 337, 161, 0, 973, 974, 1, 0, 0, 0, 974, 975, 6, 91, 17, 0, 975, 198, 1, 0, 0, 0, 976, 977, 3, 99, 42, 0, 977, 978, 1, 0, 0, 0, 978, 979, 6, 92, 18, 0, 979, 200, 1, 0, 0, 0, 980, 981, 3, 95, 40, 0, 981, 982, 1, 0, 0, 0, 982, 983, 6, 93, 19, 0, 983, 202, 1, 0, 0, 0, 984, 985, 7, 16, 0, 0, 985, 986, 7, 3, 0, 0, 986, 987, 7, 5, 0, 0, 987, 988, 7, 12, 0, 0, 988, 989, 7, 0, 0, 0, 989, 990, 7, 12, 0, 0, 990, 991, 7, 5, 0, 0, 991, 992, 7, 12, 0, 0, 992, 204, 1, 0, 0, 0, 993, 997, 8, 32, 0, 0, 994, 995, 5, 47, 0, 0, 995, 997, 8, 33, 0, 0, 996, 993, 1, 0, 0, 0, 996, 994, 1, 0, 0, 0, 997, 206, 1, 0, 0, 0, 998, 1000, 3, 205, 95, 0, 999, 998, 1, 0, 0, 0, 1000, 1001, 1, 0, 0, 0, 1001, 999, 1, 0, 0, 0, 1001, 1002, 1, 0, 0, 0, 1002, 208, 1, 0, 0, 0, 1003, 1004, 3, 207, 96, 0, 1004, 1005, 1, 0, 0, 0, 1005, 1006, 6, 97, 20, 0, 1006, 210, 1, 0, 0, 0, 1007, 1008, 3, 83, 34, 0, 1008, 1009, 1, 0, 0, 0, 1009, 1010, 6, 98, 21, 0, 1010, 212, 1, 0, 0, 0, 1011, 1012, 3, 55, 20, 0, 1012, 1013, 1, 0, 0, 0, 1013, 1014, 6, 99, 10, 0, 1014, 214, 1, 0, 0, 0, 1015, 1016, 3, 57, 21, 0, 1016, 1017, 1, 0, 0, 0, 1017, 1018, 6, 100, 10, 0, 1018, 216, 1, 0, 0, 0, 1019, 1020, 3, 59, 22, 0, 1020, 1021, 1, 0, 0, 0, 1021, 1022, 6, 101, 10, 0, 1022, 218, 1, 0, 0, 0, 1023, 1024, 3, 61, 23, 0, 1024, 1025, 1, 0, 0, 0, 1025, 1026, 6, 102, 15, 0, 1026, 1027, 6, 102, 11, 0, 1027, 220, 1, 0, 0, 0, 1028, 1029, 3, 103, 44, 0, 1029, 1030, 1, 0, 0, 0, 1030, 1031, 6, 103, 22, 0, 1031, 222, 1, 0, 0, 0, 1032, 1033, 3, 99, 42, 0, 1033, 1034, 1, 0, 0, 0, 1034, 1035, 6, 104, 18, 0, 1035, 224, 1, 0, 0, 0, 1036, 1037, 3, 127, 56, 0, 1037, 1038, 1, 0, 0, 0, 1038, 1039, 6, 105, 23, 0, 1039, 226, 1, 0, 0, 0, 1040, 1041, 3, 163, 74, 0, 1041, 1042, 1, 0, 0, 0, 1042, 1043, 6, 106, 24, 0, 1043, 228, 1, 0, 0, 0, 1044, 1049, 3, 65, 25, 0, 1045, 1049, 3, 63, 24, 0, 1046, 1049, 3, 79, 32, 0, 1047, 1049, 3, 153, 69, 0, 1048, 1044, 1, 0, 0, 0, 1048, 1045, 1, 0, 0, 0, 1048, 1046, 1, 0, 0, 0, 1048, 1047, 1, 0, 0, 0, 1049, 230, 1, 0, 0, 0, 1050, 1053, 3, 65, 25, 0, 1051, 1053, 3, 153, 69, 0, 1052, 1050, 1, 0, 0, 0, 1052, 1051, 1, 0, 0, 0, 1053, 1057, 1, 0, 0, 0, 1054, 1056, 3, 229, 107, 0, 1055, 1054, 1, 0, 0, 0, 1056, 1059, 1, 0, 0, 0, 1057, 1055, 1, 0, 0, 0, 1057, 1058, 1, 0, 0, 0, 1058, 1070, 1, 0, 0, 0, 1059, 1057, 1, 0, 0, 0, 1060, 1063, 3, 79, 32, 0, 1061, 1063, 3, 73, 29, 0, 1062, 1060, 1, 0, 0, 0, 1062, 1061, 1, 0, 0, 0, 1063, 1065, 1, 0, 0, 0, 1064, 1066, 3, 229, 107, 0, 1065, 1064, 1, 0, 0, 0, 1066, 1067, 1, 0, 0, 0, 1067, 1065, 1, 0, 0, 0, 1067, 1068, 1, 0, 0, 0, 1068, 1070, 1, 0, 0, 0, 1069, 1052, 1, 0, 0, 0, 1069, 1062, 1, 0, 0, 0, 1070, 232, 1, 0, 0, 0, 1071, 1074, 3, 231, 108, 0, 1072, 1074, 3, 171, 78, 0, 1073, 1071, 1, 0, 0, 0, 1073, 1072, 1, 0, 0, 0, 1074, 1075, 1, 0, 0, 0, 1075, 1073, 1, 0, 0, 0, 1075, 1076, 1, 0, 0, 0, 1076, 234, 1, 0, 0, 0, 1077, 1078, 3, 55, 20, 0, 1078, 1079, 1, 0, 0, 0, 1079, 1080, 6, 110, 10, 0, 1080, 236, 1, 0, 0, 0, 1081, 1082, 3, 57, 21, 0, 1082, 1083, 1, 0, 0, 0, 1083, 1084, 6, 111, 10, 0, 1084, 238, 1, 0, 0, 0, 1085, 1086, 3, 59, 22, 0, 1086, 1087, 1, 0, 0, 0, 1087, 1088, 6, 112, 10, 0, 1088, 240, 1, 0, 0, 0, 1089, 1090, 3, 61, 23, 0, 1090, 1091, 1, 0, 0, 0, 1091, 1092, 6, 113, 15, 0, 1092, 1093, 6, 113, 11, 0, 1093, 242, 1, 0, 0, 0, 1094, 1095, 3, 95, 40, 0, 1095, 1096, 1, 0, 0, 0, 1096, 1097, 6, 114, 19, 0, 1097, 244, 1, 0, 0, 0, 1098, 1099, 3, 99, 42, 0, 1099, 1100, 1, 0, 0, 0, 1100, 1101, 6, 115, 18, 0, 1101, 246, 1, 0, 0, 0, 1102, 1103, 3, 103, 44, 0, 1103, 1104, 1, 0, 0, 0, 1104, 1105, 6, 116, 22, 0, 1105, 248, 1, 0, 0, 0, 1106, 1107, 3, 127, 56, 0, 1107, 1108, 1, 0, 0, 0, 1108, 1109, 6, 117, 23, 0, 1109, 250, 1, 0, 0, 0, 1110, 1111, 3, 163, 74, 0, 1111, 1112, 1, 0, 0, 0, 1112, 1113, 6, 118, 24, 0, 1113, 252, 1, 0, 0, 0, 1114, 1115, 7, 12, 0, 0, 1115, 1116, 7, 2, 0, 0, 1116, 254, 1, 0, 0, 0, 1117, 1118, 3, 233, 109, 0, 1118, 1119, 1, 0, 0, 0, 1119, 1120, 6, 120, 25, 0, 1120, 256, 1, 0, 0, 0, 1121, 1122, 3, 55, 20, 0, 1122, 1123, 1, 0, 0, 0, 1123, 1124, 6, 121, 10, 0, 1124, 258, 1, 0, 0, 0, 1125, 1126, 3, 57, 21, 0, 1126, 1127, 1, 0, 0, 0, 1127, 1128, 6, 122, 10, 0, 1128, 260, 1, 0, 0, 0, 1129, 1130, 3, 59, 22, 0, 1130, 1131, 1, 0, 0, 0, 1131, 1132, 6, 123, 10, 0, 1132, 262, 1, 0, 0, 0, 1133, 1134, 3, 61, 23, 0, 1134, 1135, 1, 0, 0, 0, 1135, 1136, 6, 124, 15, 0, 1136, 1137, 6, 124, 11, 0, 1137, 264, 1, 0, 0, 0, 1138, 1139, 3, 165, 75, 0, 1139, 1140, 1, 0, 0, 0, 1140, 1141, 6, 125, 13, 0, 1141, 1142, 6, 125, 26, 0, 1142, 266, 1, 0, 0, 0, 1143, 1144, 7, 7, 0, 0, 1144, 1145, 7, 9, 0, 0, 1145, 1146, 1, 0, 0, 0, 1146, 1147, 6, 126, 27, 0, 1147, 268, 1, 0, 0, 0, 1148, 1149, 7, 19, 0, 0, 1149, 1150, 7, 1, 0, 0, 1150, 1151, 7, 5, 0, 0, 1151, 1152, 7, 10, 0, 0, 1152, 1153, 1, 0, 0, 0, 1153, 1154, 6, 127, 27, 0, 1154, 270, 1, 0, 0, 0, 1155, 1156, 8, 34, 0, 0, 1156, 272, 1, 0, 0, 0, 1157, 1159, 3, 271, 128, 0, 1158, 1157, 1, 0, 0, 0, 1159, 1160, 1, 0, 0, 0, 1160, 1158, 1, 0, 0, 0, 1160, 1161, 1, 0, 0, 0, 1161, 1162, 1, 0, 0, 0, 1162, 1163, 3, 337, 161, 0, 1163, 1165, 1, 0, 0, 0, 1164, 1158, 1, 0, 0, 0, 1164, 1165, 1, 0, 0, 0, 1165, 1167, 1, 0, 0, 0, 1166, 1168, 3, 271, 128, 0, 1167, 1166, 1, 0, 0, 0, 1168, 1169, 1, 0, 0, 0, 1169, 1167, 1, 0, 0, 0, 1169, 1170, 1, 0, 0, 0, 1170, 274, 1, 0, 0, 0, 1171, 1172, 3, 273, 129, 0, 1172, 1173, 1, 0, 0, 0, 1173, 1174, 6, 130, 28, 0, 1174, 276, 1, 0, 0, 0, 1175, 1176, 3, 55, 20, 0, 1176, 1177, 1, 0, 0, 0, 1177, 1178, 6, 131, 10, 0, 1178, 278, 1, 0, 0, 0, 1179, 1180, 3, 57, 21, 0, 1180, 1181, 1, 0, 0, 0, 1181, 1182, 6, 132, 10, 0, 1182, 280, 1, 0, 0, 0, 1183, 1184, 3, 59, 22, 0, 1184, 1185, 1, 0, 0, 0, 1185, 1186, 6, 133, 10, 0, 1186, 282, 1, 0, 0, 0, 1187, 1188, 3, 61, 23, 0, 1188, 1189, 1, 0, 0, 0, 1189, 1190, 6, 134, 15, 0, 1190, 1191, 6, 134, 11, 0, 1191, 1192, 6, 134, 11, 0, 1192, 284, 1, 0, 0, 0, 1193, 1194, 3, 95, 40, 0, 1194, 1195, 1, 0, 0, 0, 1195, 1196, 6, 135, 19, 0, 1196, 286, 1, 0, 0, 0, 1197, 1198, 3, 99, 42, 0, 1198, 1199, 1, 0, 0, 0, 1199, 1200, 6, 136, 18, 0, 1200, 288, 1, 0, 0, 0, 1201, 1202, 3, 103, 44, 0, 1202, 1203, 1, 0, 0, 0, 1203, 1204, 6, 137, 22, 0, 1204, 290, 1, 0, 0, 0, 1205, 1206, 3, 269, 127, 0, 1206, 1207, 1, 0, 0, 0, 1207, 1208, 6, 138, 29, 0, 1208, 292, 1, 0, 0, 0, 1209, 1210, 3, 233, 109, 0, 1210, 1211, 1, 0, 0, 0, 1211, 1212, 6, 139, 25, 0, 1212, 294, 1, 0, 0, 0, 1213, 1214, 3, 173, 79, 0, 1214, 1215, 1, 0, 0, 0, 1215, 1216, 6, 140, 30, 0, 1216, 296, 1, 0, 0, 0, 1217, 1218, 3, 127, 56, 0, 1218, 1219, 1, 0, 0, 0, 1219, 1220, 6, 141, 23, 0, 1220, 298, 1, 0, 0, 0, 1221, 1222, 3, 163, 74, 0, 1222, 1223, 1, 0, 0, 0, 1223, 1224, 6, 142, 24, 0, 1224, 300, 1, 0, 0, 0, 1225, 1226, 3, 55, 20, 0, 1226, 1227, 1, 0, 0, 0, 1227, 1228, 6, 143, 10, 0, 1228, 302, 1, 0, 0, 0, 1229, 1230, 3, 57, 21, 0, 1230, 1231, 1, 0, 0, 0, 1231, 1232, 6, 144, 10, 0, 1232, 304, 1, 0, 0, 0, 1233, 1234, 3, 59, 22, 0, 1234, 1235, 1, 0, 0, 0, 1235, 1236, 6, 145, 10, 0, 1236, 306, 1, 0, 0, 0, 1237, 1238, 3, 61, 23, 0, 1238, 1239, 1, 0, 0, 0, 1239, 1240, 6, 146, 15, 0, 1240, 1241, 6, 146, 11, 0, 1241, 308, 1, 0, 0, 0, 1242, 1243, 3, 103, 44, 0, 1243, 1244, 1, 0, 0, 0, 1244, 1245, 6, 147, 22, 0, 1245, 310, 1, 0, 0, 0, 1246, 1247, 3, 127, 56, 0, 1247, 1248, 1, 0, 0, 0, 1248, 1249, 6, 148, 23, 0, 1249, 312, 1, 0, 0, 0, 1250, 1251, 3, 163, 74, 0, 1251, 1252, 1, 0, 0, 0, 1252, 1253, 6, 149, 24, 0, 1253, 314, 1, 0, 0, 0, 1254, 1255, 3, 173, 79, 0, 1255, 1256, 1, 0, 0, 0, 1256, 1257, 6, 150, 30, 0, 1257, 316, 1, 0, 0, 0, 1258, 1259, 3, 169, 77, 0, 1259, 1260, 1, 0, 0, 0, 1260, 1261, 6, 151, 31, 0, 1261, 318, 1, 0, 0, 0, 1262, 1263, 3, 55, 20, 0, 1263, 1264, 1, 0, 0, 0, 1264, 1265, 6, 152, 10, 0, 1265, 320, 1, 0, 0, 0, 1266, 1267, 3, 57, 21, 0, 1267, 1268, 1, 0, 0, 0, 1268, 1269, 6, 153, 10, 0, 1269, 322, 1, 0, 0, 0, 1270, 1271, 3, 59, 22, 0, 1271, 1272, 1, 0, 0, 0, 1272, 1273, 6, 154, 10, 0, 1273, 324, 1, 0, 0, 0, 1274, 1275, 3, 61, 23, 0, 1275, 1276, 1, 0, 0, 0, 1276, 1277, 6, 155, 15, 0, 1277, 1278, 6, 155, 11, 0, 1278, 326, 1, 0, 0, 0, 1279, 1280, 7, 1, 0, 0, 1280, 1281, 7, 9, 0, 0, 1281, 1282, 7, 15, 0, 0, 1282, 1283, 7, 7, 0, 0, 1283, 328, 1, 0, 0, 0, 1284, 1285, 3, 55, 20, 0, 1285, 1286, 1, 0, 0, 0, 1286, 1287, 6, 157, 10, 0, 1287, 330, 1, 0, 0, 0, 1288, 1289, 3, 57, 21, 0, 1289, 1290, 1, 0, 0, 0, 1290, 1291, 6, 158, 10, 0, 1291, 332, 1, 0, 0, 0, 1292, 1293, 3, 59, 22, 0, 1293, 1294, 1, 0, 0, 0, 1294, 1295, 6, 159, 10, 0, 1295, 334, 1, 0, 0, 0, 1296, 1297, 3, 167, 76, 0, 1297, 1298, 1, 0, 0, 0, 1298, 1299, 6, 160, 16, 0, 1299, 1300, 6, 160, 11, 0, 1300, 336, 1, 0, 0, 0, 1301, 1302, 5, 58, 0, 0, 1302, 338, 1, 0, 0, 0, 1303, 1309, 3, 73, 29, 0, 1304, 1309, 3, 63, 24, 0, 1305, 1309, 3, 103, 44, 0, 1306, 1309, 3, 65, 25, 0, 1307, 1309, 3, 79, 32, 0, 1308, 1303, 1, 0, 0, 0, 1308, 1304, 1, 0, 0, 0, 1308, 1305, 1, 0, 0, 0, 1308, 1306, 1, 0, 0, 0, 1308, 1307, 1, 0, 0, 0, 1309, 1310, 1, 0, 0, 0, 1310, 1308, 1, 0, 0, 0, 1310, 1311, 1, 0, 0, 0, 1311, 340, 1, 0, 0, 0, 1312, 1313, 3, 55, 20, 0, 1313, 1314, 1, 0, 0, 0, 1314, 1315, 6, 163, 10, 0, 1315, 342, 1, 0, 0, 0, 1316, 1317, 3, 57, 21, 0, 1317, 1318, 1, 0, 0, 0, 1318, 1319, 6, 164, 10, 0, 1319, 344, 1, 0, 0, 0, 1320, 1321, 3, 59, 22, 0, 1321, 1322, 1, 0, 0, 0, 1322, 1323, 6, 165, 10, 0, 1323, 346, 1, 0, 0, 0, 1324, 1325, 3, 61, 23, 0, 1325, 1326, 1, 0, 0, 0, 1326, 1327, 6, 166, 15, 0, 1327, 1328, 6, 166, 11, 0, 1328, 348, 1, 0, 0, 0, 1329, 1330, 3, 337, 161, 0, 1330, 1331, 1, 0, 0, 0, 1331, 1332, 6, 167, 17, 0, 1332, 350, 1, 0, 0, 0, 1333, 1334, 3, 99, 42, 0, 1334, 1335, 1, 0, 0, 0, 1335, 1336, 6, 168, 18, 0, 1336, 352, 1, 0, 0, 0, 1337, 1338, 3, 103, 44, 0, 1338, 1339, 1, 0, 0, 0, 1339, 1340, 6, 169, 22, 0, 1340, 354, 1, 0, 0, 0, 1341, 1342, 3, 267, 126, 0, 1342, 1343, 1, 0, 0, 0, 1343, 1344, 6, 170, 32, 0, 1344, 1345, 6, 170, 33, 0, 1345, 356, 1, 0, 0, 0, 1346, 1347, 3, 207, 96, 0, 1347, 1348, 1, 0, 0, 0, 1348, 1349, 6, 171, 20, 0, 1349, 358, 1, 0, 0, 0, 1350, 1351, 3, 83, 34, 0, 1351, 1352, 1, 0, 0, 0, 1352, 1353, 6, 172, 21, 0, 1353, 360, 1, 0, 0, 0, 1354, 1355, 3, 55, 20, 0, 1355, 1356, 1, 0, 0, 0, 1356, 1357, 6, 173, 10, 0, 1357, 362, 1, 0, 0, 0, 1358, 1359, 3, 57, 21, 0, 1359, 1360, 1, 0, 0, 0, 1360, 1361, 6, 174, 10, 0, 1361, 364, 1, 0, 0, 0, 1362, 1363, 3, 59, 22, 0, 1363, 1364, 1, 0, 0, 0, 1364, 1365, 6, 175, 10, 0, 1365, 366, 1, 0, 0, 0, 1366, 1367, 3, 61, 23, 0, 1367, 1368, 1, 0, 0, 0, 1368, 1369, 6, 176, 15, 0, 1369, 1370, 6, 176, 11, 0, 1370, 1371, 6, 176, 11, 0, 1371, 368, 1, 0, 0, 0, 1372, 1373, 3, 99, 42, 0, 1373, 1374, 1, 0, 0, 0, 1374, 1375, 6, 177, 18, 0, 1375, 370, 1, 0, 0, 0, 1376, 1377, 3, 103, 44, 0, 1377, 1378, 1, 0, 0, 0, 1378, 1379, 6, 178, 22, 0, 1379, 372, 1, 0, 0, 0, 1380, 1381, 3, 233, 109, 0, 1381, 1382, 1, 0, 0, 0, 1382, 1383, 6, 179, 25, 0, 1383, 374, 1, 0, 0, 0, 1384, 1385, 3, 55, 20, 0, 1385, 1386, 1, 0, 0, 0, 1386, 1387, 6, 180, 10, 0, 1387, 376, 1, 0, 0, 0, 1388, 1389, 3, 57, 21, 0, 1389, 1390, 1, 0, 0, 0, 1390, 1391, 6, 181, 10, 0, 1391, 378, 1, 0, 0, 0, 1392, 1393, 3, 59, 22, 0, 1393, 1394, 1, 0, 0, 0, 1394, 1395, 6, 182, 10, 0, 1395, 380, 1, 0, 0, 0, 1396, 1397, 3, 61, 23, 0, 1397, 1398, 1, 0, 0, 0, 1398, 1399, 6, 183, 15, 0, 1399, 1400, 6, 183, 11, 0, 1400, 382, 1, 0, 0, 0, 1401, 1402, 3, 207, 96, 0, 1402, 1403, 1, 0, 0, 0, 1403, 1404, 6, 184, 20, 0, 1404, 1405, 6, 184, 11, 0, 1405, 1406, 6, 184, 34, 0, 1406, 384, 1, 0, 0, 0, 1407, 1408, 3, 83, 34, 0, 1408, 1409, 1, 0, 0, 0, 1409, 1410, 6, 185, 21, 0, 1410, 1411, 6, 185, 11, 0, 1411, 1412, 6, 185, 34, 0, 1412, 386, 1, 0, 0, 0, 1413, 1414, 3, 55, 20, 0, 1414, 1415, 1, 0, 0, 0, 1415, 1416, 6, 186, 10, 0, 1416, 388, 1, 0, 0, 0, 1417, 1418, 3, 57, 21, 0, 1418, 1419, 1, 0, 0, 0, 1419, 1420, 6, 187, 10, 0, 1420, 390, 1, 0, 0, 0, 1421, 1422, 3, 59, 22, 0, 1422, 1423, 1, 0, 0, 0, 1423, 1424, 6, 188, 10, 0, 1424, 392, 1, 0, 0, 0, 1425, 1426, 3, 337, 161, 0, 1426, 1427, 1, 0, 0, 0, 1427, 1428, 6, 189, 17, 0, 1428, 1429, 6, 189, 11, 0, 1429, 1430, 6, 189, 9, 0, 1430, 394, 1, 0, 0, 0, 1431, 1432, 3, 99, 42, 0, 1432, 1433, 1, 0, 0, 0, 1433, 1434, 6, 190, 18, 0, 1434, 1435, 6, 190, 11, 0, 1435, 1436, 6, 190, 9, 0, 1436, 396, 1, 0, 0, 0, 1437, 1438, 3, 55, 20, 0, 1438, 1439, 1, 0, 0, 0, 1439, 1440, 6, 191, 10, 0, 1440, 398, 1, 0, 0, 0, 1441, 1442, 3, 57, 21, 0, 1442, 1443, 1, 0, 0, 0, 1443, 1444, 6, 192, 10, 0, 1444, 400, 1, 0, 0, 0, 1445, 1446, 3, 59, 22, 0, 1446, 1447, 1, 0, 0, 0, 1447, 1448, 6, 193, 10, 0, 1448, 402, 1, 0, 0, 0, 1449, 1450, 3, 173, 79, 0, 1450, 1451, 1, 0, 0, 0, 1451, 1452, 6, 194, 11, 0, 1452, 1453, 6, 194, 0, 0, 1453, 1454, 6, 194, 30, 0, 1454, 404, 1, 0, 0, 0, 1455, 1456, 3, 169, 77, 0, 1456, 1457, 1, 0, 0, 0, 1457, 1458, 6, 195, 11, 0, 1458, 1459, 6, 195, 0, 0, 1459, 1460, 6, 195, 31, 0, 1460, 406, 1, 0, 0, 0, 1461, 1462, 3, 89, 37, 0, 1462, 1463, 1, 0, 0, 0, 1463, 1464, 6, 196, 11, 0, 1464, 1465, 6, 196, 0, 0, 1465, 1466, 6, 196, 35, 0, 1466, 408, 1, 0, 0, 0, 1467, 1468, 3, 61, 23, 0, 1468, 1469, 1, 0, 0, 0, 1469, 1470, 6, 197, 15, 0, 1470, 1471, 6, 197, 11, 0, 1471, 410, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 579, 589, 593, 596, 605, 607, 618, 637, 642, 651, 658, 663, 665, 676, 684, 687, 689, 694, 699, 705, 712, 717, 723, 726, 734, 738, 871, 876, 883, 885, 901, 906, 911, 913, 919, 996, 1001, 1048, 1052, 1057, 1062, 1067, 1069, 1073, 1075, 1160, 1164, 1169, 1308, 1310, 36, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 65, 0, 5, 0, 0, 7, 24, 0, 7, 66, 0, 7, 104, 0, 7, 33, 0, 7, 31, 0, 7, 76, 0, 7, 25, 0, 7, 35, 0, 7, 47, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 28, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index f67daa29ab059..3bef23f4d2751 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -27,13 +27,13 @@ public class EsqlBaseLexer extends LexerConfig { public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, - UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, - QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, - ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, - IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, - PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, - GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, + WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, UNKNOWN_CMD=20, + LINE_COMMENT=21, MULTILINE_COMMENT=22, WS=23, PIPE=24, QUOTED_STRING=25, + INTEGER_LITERAL=26, DECIMAL_LITERAL=27, BY=28, AND=29, ASC=30, ASSIGN=31, + CAST_OP=32, COMMA=33, DESC=34, DOT=35, FALSE=36, FIRST=37, IN=38, IS=39, + LAST=40, LIKE=41, LP=42, NOT=43, NULL=44, NULLS=45, OR=46, PARAM=47, RLIKE=48, + RP=49, TRUE=50, EQ=51, CIEQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, MATCH=63, NAMED_OR_POSITIONAL_PARAM=64, OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, @@ -68,15 +68,15 @@ private static String[] makeRuleNames() { return new String[] { "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", - "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", "UNKNOWN_CMD", - "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", - "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", - "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "DEV_MATCH_OP", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", + "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", "LINE_COMMENT", + "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", + "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", + "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", + "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", + "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", + "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", + "PERCENT", "MATCH", "NESTED_WHERE", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", @@ -119,15 +119,15 @@ private static String[] makeLiteralNames() { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, - null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", - "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", - "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", - "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", - "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, - null, null, null, null, null, "'metadata'", null, null, null, null, null, - null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, - null, null, null, null, null, null, null, null, "'info'", null, null, - null, "':'" + "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", + "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", + "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", + "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", + "'-'", "'*'", "'/'", "'%'", "'match'", null, null, "']'", null, null, + null, null, null, null, null, null, "'metadata'", null, null, null, null, + null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, + null, null, null, null, null, null, null, null, null, "'info'", null, + null, null, "':'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -135,13 +135,13 @@ private static String[] makeSymbolicNames() { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", - "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", + "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", + "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", + "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", + "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", + "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", + "PERCENT", "MATCH", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", @@ -226,11 +226,9 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { case 17: return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); case 18: - return DEV_MATCH_sempred((RuleContext)_localctx, predIndex); - case 19: return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); case 73: - return DEV_MATCH_OP_sempred((RuleContext)_localctx, predIndex); + return NESTED_WHERE_sempred((RuleContext)_localctx, predIndex); } return true; } @@ -248,30 +246,23 @@ private boolean DEV_LOOKUP_sempred(RuleContext _localctx, int predIndex) { } return true; } - private boolean DEV_MATCH_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 2: return this.isDevVersion(); } return true; } - private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { + private boolean NESTED_WHERE_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 3: return this.isDevVersion(); } return true; } - private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 4: - return this.isDevVersion(); - } - return true; - } public static final String _serializedATN = - "\u0004\u0000x\u05c3\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0004\u0000x\u05c0\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ @@ -352,883 +343,881 @@ private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) { "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011"+ "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ "\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+ - "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ - "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+ - "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0004\u0014\u024b\b\u0014"+ - "\u000b\u0014\f\u0014\u024c\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+ - "\u0001\u0015\u0001\u0015\u0005\u0015\u0255\b\u0015\n\u0015\f\u0015\u0258"+ - "\t\u0015\u0001\u0015\u0003\u0015\u025b\b\u0015\u0001\u0015\u0003\u0015"+ - "\u025e\b\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+ - "\u0001\u0016\u0001\u0016\u0005\u0016\u0267\b\u0016\n\u0016\f\u0016\u026a"+ - "\t\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001"+ - "\u0017\u0004\u0017\u0272\b\u0017\u000b\u0017\f\u0017\u0273\u0001\u0017"+ - "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"+ - "\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+ - "\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0003\u001d\u0287\b\u001d"+ - "\u0001\u001d\u0004\u001d\u028a\b\u001d\u000b\u001d\f\u001d\u028b\u0001"+ - "\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001 \u0003"+ - " \u0295\b \u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0003\"\u029c\b\"\u0001"+ - "#\u0001#\u0001#\u0005#\u02a1\b#\n#\f#\u02a4\t#\u0001#\u0001#\u0001#\u0001"+ - "#\u0001#\u0001#\u0005#\u02ac\b#\n#\f#\u02af\t#\u0001#\u0001#\u0001#\u0001"+ - "#\u0001#\u0003#\u02b6\b#\u0001#\u0003#\u02b9\b#\u0003#\u02bb\b#\u0001"+ - "$\u0004$\u02be\b$\u000b$\f$\u02bf\u0001%\u0004%\u02c3\b%\u000b%\f%\u02c4"+ - "\u0001%\u0001%\u0005%\u02c9\b%\n%\f%\u02cc\t%\u0001%\u0001%\u0004%\u02d0"+ - "\b%\u000b%\f%\u02d1\u0001%\u0004%\u02d5\b%\u000b%\f%\u02d6\u0001%\u0001"+ - "%\u0005%\u02db\b%\n%\f%\u02de\t%\u0003%\u02e0\b%\u0001%\u0001%\u0001%"+ - "\u0001%\u0004%\u02e6\b%\u000b%\f%\u02e7\u0001%\u0001%\u0003%\u02ec\b%"+ - "\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001"+ - "(\u0001(\u0001)\u0001)\u0001*\u0001*\u0001*\u0001+\u0001+\u0001,\u0001"+ - ",\u0001,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001"+ - ".\u0001.\u0001/\u0001/\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u0001"+ - "0\u00011\u00011\u00011\u00012\u00012\u00012\u00012\u00012\u00013\u0001"+ - "3\u00013\u00013\u00013\u00014\u00014\u00015\u00015\u00015\u00015\u0001"+ - "6\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u00017\u0001"+ - "7\u00018\u00018\u00018\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001"+ - ":\u0001:\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001"+ - "=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001"+ - "A\u0001A\u0001A\u0001B\u0001B\u0001C\u0001C\u0001C\u0001D\u0001D\u0001"+ - "E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001"+ - "I\u0001I\u0001I\u0001J\u0001J\u0001J\u0003J\u036b\bJ\u0001J\u0005J\u036e"+ - "\bJ\nJ\fJ\u0371\tJ\u0001J\u0001J\u0004J\u0375\bJ\u000bJ\fJ\u0376\u0003"+ - "J\u0379\bJ\u0001K\u0001K\u0001K\u0001K\u0001K\u0001L\u0001L\u0001L\u0001"+ - "L\u0001L\u0001M\u0001M\u0005M\u0387\bM\nM\fM\u038a\tM\u0001M\u0001M\u0003"+ - "M\u038e\bM\u0001M\u0004M\u0391\bM\u000bM\fM\u0392\u0003M\u0395\bM\u0001"+ - "N\u0001N\u0004N\u0399\bN\u000bN\fN\u039a\u0001N\u0001N\u0001O\u0001O\u0001"+ - "P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001"+ - "R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001"+ - "T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001"+ - "W\u0001W\u0001W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001"+ - "Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001"+ - "[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001"+ - "^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001"+ - "_\u0003_\u03e8\b_\u0001`\u0004`\u03eb\b`\u000b`\f`\u03ec\u0001a\u0001"+ - "a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001"+ - "c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001"+ - "f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001"+ - "h\u0001h\u0001i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001"+ - "k\u0001k\u0001k\u0001k\u0003k\u041c\bk\u0001l\u0001l\u0003l\u0420\bl\u0001"+ - "l\u0005l\u0423\bl\nl\fl\u0426\tl\u0001l\u0001l\u0003l\u042a\bl\u0001l"+ - "\u0004l\u042d\bl\u000bl\fl\u042e\u0003l\u0431\bl\u0001m\u0001m\u0004m"+ - "\u0435\bm\u000bm\fm\u0436\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001"+ - "o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001"+ - "q\u0001r\u0001r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001"+ - "t\u0001t\u0001t\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001"+ - "v\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+ - "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+ - "|\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001}\u0001"+ - "~\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001"+ - "\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001"+ - "\u0081\u0004\u0081\u048a\b\u0081\u000b\u0081\f\u0081\u048b\u0001\u0081"+ - "\u0001\u0081\u0003\u0081\u0490\b\u0081\u0001\u0081\u0004\u0081\u0493\b"+ - "\u0081\u000b\u0081\f\u0081\u0494\u0001\u0082\u0001\u0082\u0001\u0082\u0001"+ - "\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001"+ - "\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001"+ - "\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001"+ - "\u0086\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001"+ - "\u0088\u0001\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001"+ - "\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001"+ - "\u008b\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+ - "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001"+ - "\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001"+ - "\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001"+ - "\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001"+ - "\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001"+ - "\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001"+ - "\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001"+ - "\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001"+ - "\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001"+ - "\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001"+ - "\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001"+ - "\u009c\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001"+ - "\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001"+ - "\u009f\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001"+ - "\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001"+ - "\u00a2\u0001\u00a2\u0004\u00a2\u0520\b\u00a2\u000b\u00a2\f\u00a2\u0521"+ - "\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4"+ - "\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5"+ - "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7"+ - "\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8"+ - "\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa"+ - "\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab"+ - "\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac"+ - "\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae"+ - "\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af"+ - "\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0"+ - "\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2"+ - "\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3"+ - "\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5"+ - "\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6"+ - "\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8"+ - "\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9"+ - "\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba"+ - "\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb"+ - "\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd"+ - "\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be"+ - "\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf"+ - "\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0"+ - "\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2"+ - "\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3"+ - "\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4"+ - "\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5"+ - "\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0002\u0268\u02ad\u0000"+ - "\u00c6\u000f\u0001\u0011\u0002\u0013\u0003\u0015\u0004\u0017\u0005\u0019"+ - "\u0006\u001b\u0007\u001d\b\u001f\t!\n#\u000b%\f\'\r)\u000e+\u000f-\u0010"+ - "/\u00111\u00123\u00135\u00147\u00159\u0016;\u0017=\u0018?\u0019A\u0000"+ - "C\u0000E\u0000G\u0000I\u0000K\u0000M\u0000O\u0000Q\u0000S\u0000U\u001a"+ - "W\u001bY\u001c[\u001d]\u001e_\u001fa c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f"+ - "/\u00810\u00831\u00852\u00873\u00894\u008b5\u008d6\u008f7\u00918\u0093"+ - "9\u0095:\u0097;\u0099<\u009b=\u009d>\u009f?\u00a1\u0000\u00a3@\u00a5A"+ - "\u00a7B\u00a9C\u00ab\u0000\u00adD\u00afE\u00b1F\u00b3G\u00b5\u0000\u00b7"+ - "\u0000\u00b9H\u00bbI\u00bdJ\u00bf\u0000\u00c1\u0000\u00c3\u0000\u00c5"+ - "\u0000\u00c7\u0000\u00c9\u0000\u00cbK\u00cd\u0000\u00cfL\u00d1\u0000\u00d3"+ - "\u0000\u00d5M\u00d7N\u00d9O\u00db\u0000\u00dd\u0000\u00df\u0000\u00e1"+ - "\u0000\u00e3\u0000\u00e5\u0000\u00e7\u0000\u00e9P\u00ebQ\u00edR\u00ef"+ - "S\u00f1\u0000\u00f3\u0000\u00f5\u0000\u00f7\u0000\u00f9\u0000\u00fb\u0000"+ - "\u00fdT\u00ff\u0000\u0101U\u0103V\u0105W\u0107\u0000\u0109\u0000\u010b"+ - "X\u010dY\u010f\u0000\u0111Z\u0113\u0000\u0115[\u0117\\\u0119]\u011b\u0000"+ - "\u011d\u0000\u011f\u0000\u0121\u0000\u0123\u0000\u0125\u0000\u0127\u0000"+ - "\u0129\u0000\u012b\u0000\u012d^\u012f_\u0131`\u0133\u0000\u0135\u0000"+ - "\u0137\u0000\u0139\u0000\u013b\u0000\u013d\u0000\u013fa\u0141b\u0143c"+ - "\u0145\u0000\u0147d\u0149e\u014bf\u014dg\u014f\u0000\u0151h\u0153i\u0155"+ - "j\u0157k\u0159l\u015b\u0000\u015d\u0000\u015f\u0000\u0161\u0000\u0163"+ - "\u0000\u0165\u0000\u0167\u0000\u0169m\u016bn\u016do\u016f\u0000\u0171"+ - "\u0000\u0173\u0000\u0175\u0000\u0177p\u0179q\u017br\u017d\u0000\u017f"+ - "\u0000\u0181\u0000\u0183s\u0185t\u0187u\u0189\u0000\u018b\u0000\u018d"+ - "v\u018fw\u0191x\u0193\u0000\u0195\u0000\u0197\u0000\u0199\u0000\u000f"+ - "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+ - "#\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002"+ - "\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000"+ - "PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002"+ - "\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000"+ - "GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r"+ - "\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002"+ - "\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002"+ - "\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t"+ - "\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r \"#,,"+ - "//::<<>?\\\\||\u05df\u0000\u000f\u0001\u0000\u0000\u0000\u0000\u0011\u0001"+ - "\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000\u0000\u0015\u0001"+ - "\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000\u0000\u0019\u0001"+ - "\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000\u0000\u001d\u0001"+ - "\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000\u0000!\u0001\u0000"+ - "\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%\u0001\u0000\u0000\u0000"+ - "\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001\u0000\u0000\u0000\u0000"+ - "+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000\u0000\u0000/\u0001"+ - "\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u00003\u0001\u0000\u0000"+ - "\u0000\u00005\u0001\u0000\u0000\u0000\u00007\u0001\u0000\u0000\u0000\u0000"+ - "9\u0001\u0000\u0000\u0000\u0000;\u0001\u0000\u0000\u0000\u0000=\u0001"+ - "\u0000\u0000\u0000\u0001?\u0001\u0000\u0000\u0000\u0001U\u0001\u0000\u0000"+ - "\u0000\u0001W\u0001\u0000\u0000\u0000\u0001Y\u0001\u0000\u0000\u0000\u0001"+ - "[\u0001\u0000\u0000\u0000\u0001]\u0001\u0000\u0000\u0000\u0001_\u0001"+ - "\u0000\u0000\u0000\u0001a\u0001\u0000\u0000\u0000\u0001c\u0001\u0000\u0000"+ - "\u0000\u0001e\u0001\u0000\u0000\u0000\u0001g\u0001\u0000\u0000\u0000\u0001"+ - "i\u0001\u0000\u0000\u0000\u0001k\u0001\u0000\u0000\u0000\u0001m\u0001"+ - "\u0000\u0000\u0000\u0001o\u0001\u0000\u0000\u0000\u0001q\u0001\u0000\u0000"+ - "\u0000\u0001s\u0001\u0000\u0000\u0000\u0001u\u0001\u0000\u0000\u0000\u0001"+ - "w\u0001\u0000\u0000\u0000\u0001y\u0001\u0000\u0000\u0000\u0001{\u0001"+ - "\u0000\u0000\u0000\u0001}\u0001\u0000\u0000\u0000\u0001\u007f\u0001\u0000"+ - "\u0000\u0000\u0001\u0081\u0001\u0000\u0000\u0000\u0001\u0083\u0001\u0000"+ - "\u0000\u0000\u0001\u0085\u0001\u0000\u0000\u0000\u0001\u0087\u0001\u0000"+ - "\u0000\u0000\u0001\u0089\u0001\u0000\u0000\u0000\u0001\u008b\u0001\u0000"+ - "\u0000\u0000\u0001\u008d\u0001\u0000\u0000\u0000\u0001\u008f\u0001\u0000"+ - "\u0000\u0000\u0001\u0091\u0001\u0000\u0000\u0000\u0001\u0093\u0001\u0000"+ - "\u0000\u0000\u0001\u0095\u0001\u0000\u0000\u0000\u0001\u0097\u0001\u0000"+ - "\u0000\u0000\u0001\u0099\u0001\u0000\u0000\u0000\u0001\u009b\u0001\u0000"+ - "\u0000\u0000\u0001\u009d\u0001\u0000\u0000\u0000\u0001\u009f\u0001\u0000"+ - "\u0000\u0000\u0001\u00a1\u0001\u0000\u0000\u0000\u0001\u00a3\u0001\u0000"+ - "\u0000\u0000\u0001\u00a5\u0001\u0000\u0000\u0000\u0001\u00a7\u0001\u0000"+ - "\u0000\u0000\u0001\u00a9\u0001\u0000\u0000\u0000\u0001\u00ad\u0001\u0000"+ - "\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001\u00b1\u0001\u0000"+ - "\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0002\u00b5\u0001\u0000"+ - "\u0000\u0000\u0002\u00b7\u0001\u0000\u0000\u0000\u0002\u00b9\u0001\u0000"+ - "\u0000\u0000\u0002\u00bb\u0001\u0000\u0000\u0000\u0002\u00bd\u0001\u0000"+ - "\u0000\u0000\u0003\u00bf\u0001\u0000\u0000\u0000\u0003\u00c1\u0001\u0000"+ - "\u0000\u0000\u0003\u00c3\u0001\u0000\u0000\u0000\u0003\u00c5\u0001\u0000"+ - "\u0000\u0000\u0003\u00c7\u0001\u0000\u0000\u0000\u0003\u00c9\u0001\u0000"+ - "\u0000\u0000\u0003\u00cb\u0001\u0000\u0000\u0000\u0003\u00cf\u0001\u0000"+ - "\u0000\u0000\u0003\u00d1\u0001\u0000\u0000\u0000\u0003\u00d3\u0001\u0000"+ - "\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003\u00d7\u0001\u0000"+ - "\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0004\u00db\u0001\u0000"+ - "\u0000\u0000\u0004\u00dd\u0001\u0000\u0000\u0000\u0004\u00df\u0001\u0000"+ - "\u0000\u0000\u0004\u00e1\u0001\u0000\u0000\u0000\u0004\u00e3\u0001\u0000"+ - "\u0000\u0000\u0004\u00e9\u0001\u0000\u0000\u0000\u0004\u00eb\u0001\u0000"+ - "\u0000\u0000\u0004\u00ed\u0001\u0000\u0000\u0000\u0004\u00ef\u0001\u0000"+ - "\u0000\u0000\u0005\u00f1\u0001\u0000\u0000\u0000\u0005\u00f3\u0001\u0000"+ - "\u0000\u0000\u0005\u00f5\u0001\u0000\u0000\u0000\u0005\u00f7\u0001\u0000"+ - "\u0000\u0000\u0005\u00f9\u0001\u0000\u0000\u0000\u0005\u00fb\u0001\u0000"+ - "\u0000\u0000\u0005\u00fd\u0001\u0000\u0000\u0000\u0005\u00ff\u0001\u0000"+ - "\u0000\u0000\u0005\u0101\u0001\u0000\u0000\u0000\u0005\u0103\u0001\u0000"+ - "\u0000\u0000\u0005\u0105\u0001\u0000\u0000\u0000\u0006\u0107\u0001\u0000"+ - "\u0000\u0000\u0006\u0109\u0001\u0000\u0000\u0000\u0006\u010b\u0001\u0000"+ - "\u0000\u0000\u0006\u010d\u0001\u0000\u0000\u0000\u0006\u0111\u0001\u0000"+ - "\u0000\u0000\u0006\u0113\u0001\u0000\u0000\u0000\u0006\u0115\u0001\u0000"+ - "\u0000\u0000\u0006\u0117\u0001\u0000\u0000\u0000\u0006\u0119\u0001\u0000"+ - "\u0000\u0000\u0007\u011b\u0001\u0000\u0000\u0000\u0007\u011d\u0001\u0000"+ - "\u0000\u0000\u0007\u011f\u0001\u0000\u0000\u0000\u0007\u0121\u0001\u0000"+ - "\u0000\u0000\u0007\u0123\u0001\u0000\u0000\u0000\u0007\u0125\u0001\u0000"+ - "\u0000\u0000\u0007\u0127\u0001\u0000\u0000\u0000\u0007\u0129\u0001\u0000"+ - "\u0000\u0000\u0007\u012b\u0001\u0000\u0000\u0000\u0007\u012d\u0001\u0000"+ - "\u0000\u0000\u0007\u012f\u0001\u0000\u0000\u0000\u0007\u0131\u0001\u0000"+ - "\u0000\u0000\b\u0133\u0001\u0000\u0000\u0000\b\u0135\u0001\u0000\u0000"+ - "\u0000\b\u0137\u0001\u0000\u0000\u0000\b\u0139\u0001\u0000\u0000\u0000"+ - "\b\u013b\u0001\u0000\u0000\u0000\b\u013d\u0001\u0000\u0000\u0000\b\u013f"+ - "\u0001\u0000\u0000\u0000\b\u0141\u0001\u0000\u0000\u0000\b\u0143\u0001"+ - "\u0000\u0000\u0000\t\u0145\u0001\u0000\u0000\u0000\t\u0147\u0001\u0000"+ - "\u0000\u0000\t\u0149\u0001\u0000\u0000\u0000\t\u014b\u0001\u0000\u0000"+ - "\u0000\t\u014d\u0001\u0000\u0000\u0000\n\u014f\u0001\u0000\u0000\u0000"+ - "\n\u0151\u0001\u0000\u0000\u0000\n\u0153\u0001\u0000\u0000\u0000\n\u0155"+ - "\u0001\u0000\u0000\u0000\n\u0157\u0001\u0000\u0000\u0000\n\u0159\u0001"+ - "\u0000\u0000\u0000\u000b\u015b\u0001\u0000\u0000\u0000\u000b\u015d\u0001"+ - "\u0000\u0000\u0000\u000b\u015f\u0001\u0000\u0000\u0000\u000b\u0161\u0001"+ - "\u0000\u0000\u0000\u000b\u0163\u0001\u0000\u0000\u0000\u000b\u0165\u0001"+ - "\u0000\u0000\u0000\u000b\u0167\u0001\u0000\u0000\u0000\u000b\u0169\u0001"+ - "\u0000\u0000\u0000\u000b\u016b\u0001\u0000\u0000\u0000\u000b\u016d\u0001"+ - "\u0000\u0000\u0000\f\u016f\u0001\u0000\u0000\u0000\f\u0171\u0001\u0000"+ - "\u0000\u0000\f\u0173\u0001\u0000\u0000\u0000\f\u0175\u0001\u0000\u0000"+ - "\u0000\f\u0177\u0001\u0000\u0000\u0000\f\u0179\u0001\u0000\u0000\u0000"+ - "\f\u017b\u0001\u0000\u0000\u0000\r\u017d\u0001\u0000\u0000\u0000\r\u017f"+ - "\u0001\u0000\u0000\u0000\r\u0181\u0001\u0000\u0000\u0000\r\u0183\u0001"+ - "\u0000\u0000\u0000\r\u0185\u0001\u0000\u0000\u0000\r\u0187\u0001\u0000"+ - "\u0000\u0000\u000e\u0189\u0001\u0000\u0000\u0000\u000e\u018b\u0001\u0000"+ - "\u0000\u0000\u000e\u018d\u0001\u0000\u0000\u0000\u000e\u018f\u0001\u0000"+ - "\u0000\u0000\u000e\u0191\u0001\u0000\u0000\u0000\u000e\u0193\u0001\u0000"+ - "\u0000\u0000\u000e\u0195\u0001\u0000\u0000\u0000\u000e\u0197\u0001\u0000"+ - "\u0000\u0000\u000e\u0199\u0001\u0000\u0000\u0000\u000f\u019b\u0001\u0000"+ - "\u0000\u0000\u0011\u01a5\u0001\u0000\u0000\u0000\u0013\u01ac\u0001\u0000"+ - "\u0000\u0000\u0015\u01b5\u0001\u0000\u0000\u0000\u0017\u01bc\u0001\u0000"+ - "\u0000\u0000\u0019\u01c6\u0001\u0000\u0000\u0000\u001b\u01cd\u0001\u0000"+ - "\u0000\u0000\u001d\u01d4\u0001\u0000\u0000\u0000\u001f\u01db\u0001\u0000"+ - "\u0000\u0000!\u01e3\u0001\u0000\u0000\u0000#\u01ef\u0001\u0000\u0000\u0000"+ - "%\u01f8\u0001\u0000\u0000\u0000\'\u01fe\u0001\u0000\u0000\u0000)\u0205"+ - "\u0001\u0000\u0000\u0000+\u020c\u0001\u0000\u0000\u0000-\u0214\u0001\u0000"+ - "\u0000\u0000/\u021c\u0001\u0000\u0000\u00001\u022b\u0001\u0000\u0000\u0000"+ - "3\u0235\u0001\u0000\u0000\u00005\u023e\u0001\u0000\u0000\u00007\u024a"+ - "\u0001\u0000\u0000\u00009\u0250\u0001\u0000\u0000\u0000;\u0261\u0001\u0000"+ - "\u0000\u0000=\u0271\u0001\u0000\u0000\u0000?\u0277\u0001\u0000\u0000\u0000"+ - "A\u027b\u0001\u0000\u0000\u0000C\u027d\u0001\u0000\u0000\u0000E\u027f"+ - "\u0001\u0000\u0000\u0000G\u0282\u0001\u0000\u0000\u0000I\u0284\u0001\u0000"+ - "\u0000\u0000K\u028d\u0001\u0000\u0000\u0000M\u028f\u0001\u0000\u0000\u0000"+ - "O\u0294\u0001\u0000\u0000\u0000Q\u0296\u0001\u0000\u0000\u0000S\u029b"+ - "\u0001\u0000\u0000\u0000U\u02ba\u0001\u0000\u0000\u0000W\u02bd\u0001\u0000"+ - "\u0000\u0000Y\u02eb\u0001\u0000\u0000\u0000[\u02ed\u0001\u0000\u0000\u0000"+ - "]\u02f0\u0001\u0000\u0000\u0000_\u02f4\u0001\u0000\u0000\u0000a\u02f8"+ - "\u0001\u0000\u0000\u0000c\u02fa\u0001\u0000\u0000\u0000e\u02fd\u0001\u0000"+ - "\u0000\u0000g\u02ff\u0001\u0000\u0000\u0000i\u0304\u0001\u0000\u0000\u0000"+ - "k\u0306\u0001\u0000\u0000\u0000m\u030c\u0001\u0000\u0000\u0000o\u0312"+ - "\u0001\u0000\u0000\u0000q\u0315\u0001\u0000\u0000\u0000s\u0318\u0001\u0000"+ - "\u0000\u0000u\u031d\u0001\u0000\u0000\u0000w\u0322\u0001\u0000\u0000\u0000"+ - "y\u0324\u0001\u0000\u0000\u0000{\u0328\u0001\u0000\u0000\u0000}\u032d"+ - "\u0001\u0000\u0000\u0000\u007f\u0333\u0001\u0000\u0000\u0000\u0081\u0336"+ - "\u0001\u0000\u0000\u0000\u0083\u0338\u0001\u0000\u0000\u0000\u0085\u033e"+ - "\u0001\u0000\u0000\u0000\u0087\u0340\u0001\u0000\u0000\u0000\u0089\u0345"+ - "\u0001\u0000\u0000\u0000\u008b\u0348\u0001\u0000\u0000\u0000\u008d\u034b"+ - "\u0001\u0000\u0000\u0000\u008f\u034e\u0001\u0000\u0000\u0000\u0091\u0350"+ - "\u0001\u0000\u0000\u0000\u0093\u0353\u0001\u0000\u0000\u0000\u0095\u0355"+ - "\u0001\u0000\u0000\u0000\u0097\u0358\u0001\u0000\u0000\u0000\u0099\u035a"+ - "\u0001\u0000\u0000\u0000\u009b\u035c\u0001\u0000\u0000\u0000\u009d\u035e"+ - "\u0001\u0000\u0000\u0000\u009f\u0360\u0001\u0000\u0000\u0000\u00a1\u0362"+ - "\u0001\u0000\u0000\u0000\u00a3\u0378\u0001\u0000\u0000\u0000\u00a5\u037a"+ - "\u0001\u0000\u0000\u0000\u00a7\u037f\u0001\u0000\u0000\u0000\u00a9\u0394"+ - "\u0001\u0000\u0000\u0000\u00ab\u0396\u0001\u0000\u0000\u0000\u00ad\u039e"+ - "\u0001\u0000\u0000\u0000\u00af\u03a0\u0001\u0000\u0000\u0000\u00b1\u03a4"+ - "\u0001\u0000\u0000\u0000\u00b3\u03a8\u0001\u0000\u0000\u0000\u00b5\u03ac"+ - "\u0001\u0000\u0000\u0000\u00b7\u03b1\u0001\u0000\u0000\u0000\u00b9\u03b6"+ - "\u0001\u0000\u0000\u0000\u00bb\u03ba\u0001\u0000\u0000\u0000\u00bd\u03be"+ - "\u0001\u0000\u0000\u0000\u00bf\u03c2\u0001\u0000\u0000\u0000\u00c1\u03c7"+ - "\u0001\u0000\u0000\u0000\u00c3\u03cb\u0001\u0000\u0000\u0000\u00c5\u03cf"+ - "\u0001\u0000\u0000\u0000\u00c7\u03d3\u0001\u0000\u0000\u0000\u00c9\u03d7"+ - "\u0001\u0000\u0000\u0000\u00cb\u03db\u0001\u0000\u0000\u0000\u00cd\u03e7"+ - "\u0001\u0000\u0000\u0000\u00cf\u03ea\u0001\u0000\u0000\u0000\u00d1\u03ee"+ - "\u0001\u0000\u0000\u0000\u00d3\u03f2\u0001\u0000\u0000\u0000\u00d5\u03f6"+ - "\u0001\u0000\u0000\u0000\u00d7\u03fa\u0001\u0000\u0000\u0000\u00d9\u03fe"+ - "\u0001\u0000\u0000\u0000\u00db\u0402\u0001\u0000\u0000\u0000\u00dd\u0407"+ - "\u0001\u0000\u0000\u0000\u00df\u040b\u0001\u0000\u0000\u0000\u00e1\u040f"+ - "\u0001\u0000\u0000\u0000\u00e3\u0413\u0001\u0000\u0000\u0000\u00e5\u041b"+ - "\u0001\u0000\u0000\u0000\u00e7\u0430\u0001\u0000\u0000\u0000\u00e9\u0434"+ - "\u0001\u0000\u0000\u0000\u00eb\u0438\u0001\u0000\u0000\u0000\u00ed\u043c"+ - "\u0001\u0000\u0000\u0000\u00ef\u0440\u0001\u0000\u0000\u0000\u00f1\u0444"+ - "\u0001\u0000\u0000\u0000\u00f3\u0449\u0001\u0000\u0000\u0000\u00f5\u044d"+ - "\u0001\u0000\u0000\u0000\u00f7\u0451\u0001\u0000\u0000\u0000\u00f9\u0455"+ - "\u0001\u0000\u0000\u0000\u00fb\u0459\u0001\u0000\u0000\u0000\u00fd\u045d"+ - "\u0001\u0000\u0000\u0000\u00ff\u0460\u0001\u0000\u0000\u0000\u0101\u0464"+ - "\u0001\u0000\u0000\u0000\u0103\u0468\u0001\u0000\u0000\u0000\u0105\u046c"+ - "\u0001\u0000\u0000\u0000\u0107\u0470\u0001\u0000\u0000\u0000\u0109\u0475"+ - "\u0001\u0000\u0000\u0000\u010b\u047a\u0001\u0000\u0000\u0000\u010d\u047f"+ - "\u0001\u0000\u0000\u0000\u010f\u0486\u0001\u0000\u0000\u0000\u0111\u048f"+ - "\u0001\u0000\u0000\u0000\u0113\u0496\u0001\u0000\u0000\u0000\u0115\u049a"+ - "\u0001\u0000\u0000\u0000\u0117\u049e\u0001\u0000\u0000\u0000\u0119\u04a2"+ - "\u0001\u0000\u0000\u0000\u011b\u04a6\u0001\u0000\u0000\u0000\u011d\u04ac"+ - "\u0001\u0000\u0000\u0000\u011f\u04b0\u0001\u0000\u0000\u0000\u0121\u04b4"+ - "\u0001\u0000\u0000\u0000\u0123\u04b8\u0001\u0000\u0000\u0000\u0125\u04bc"+ - "\u0001\u0000\u0000\u0000\u0127\u04c0\u0001\u0000\u0000\u0000\u0129\u04c4"+ - "\u0001\u0000\u0000\u0000\u012b\u04c8\u0001\u0000\u0000\u0000\u012d\u04cc"+ - "\u0001\u0000\u0000\u0000\u012f\u04d0\u0001\u0000\u0000\u0000\u0131\u04d4"+ - "\u0001\u0000\u0000\u0000\u0133\u04d8\u0001\u0000\u0000\u0000\u0135\u04dd"+ - "\u0001\u0000\u0000\u0000\u0137\u04e1\u0001\u0000\u0000\u0000\u0139\u04e5"+ - "\u0001\u0000\u0000\u0000\u013b\u04e9\u0001\u0000\u0000\u0000\u013d\u04ed"+ - "\u0001\u0000\u0000\u0000\u013f\u04f1\u0001\u0000\u0000\u0000\u0141\u04f5"+ - "\u0001\u0000\u0000\u0000\u0143\u04f9\u0001\u0000\u0000\u0000\u0145\u04fd"+ - "\u0001\u0000\u0000\u0000\u0147\u0502\u0001\u0000\u0000\u0000\u0149\u0507"+ - "\u0001\u0000\u0000\u0000\u014b\u050b\u0001\u0000\u0000\u0000\u014d\u050f"+ - "\u0001\u0000\u0000\u0000\u014f\u0513\u0001\u0000\u0000\u0000\u0151\u0518"+ - "\u0001\u0000\u0000\u0000\u0153\u051f\u0001\u0000\u0000\u0000\u0155\u0523"+ - "\u0001\u0000\u0000\u0000\u0157\u0527\u0001\u0000\u0000\u0000\u0159\u052b"+ - "\u0001\u0000\u0000\u0000\u015b\u052f\u0001\u0000\u0000\u0000\u015d\u0534"+ - "\u0001\u0000\u0000\u0000\u015f\u0538\u0001\u0000\u0000\u0000\u0161\u053c"+ - "\u0001\u0000\u0000\u0000\u0163\u0540\u0001\u0000\u0000\u0000\u0165\u0545"+ - "\u0001\u0000\u0000\u0000\u0167\u0549\u0001\u0000\u0000\u0000\u0169\u054d"+ - "\u0001\u0000\u0000\u0000\u016b\u0551\u0001\u0000\u0000\u0000\u016d\u0555"+ - "\u0001\u0000\u0000\u0000\u016f\u0559\u0001\u0000\u0000\u0000\u0171\u055f"+ - "\u0001\u0000\u0000\u0000\u0173\u0563\u0001\u0000\u0000\u0000\u0175\u0567"+ - "\u0001\u0000\u0000\u0000\u0177\u056b\u0001\u0000\u0000\u0000\u0179\u056f"+ - "\u0001\u0000\u0000\u0000\u017b\u0573\u0001\u0000\u0000\u0000\u017d\u0577"+ - "\u0001\u0000\u0000\u0000\u017f\u057c\u0001\u0000\u0000\u0000\u0181\u0582"+ - "\u0001\u0000\u0000\u0000\u0183\u0588\u0001\u0000\u0000\u0000\u0185\u058c"+ - "\u0001\u0000\u0000\u0000\u0187\u0590\u0001\u0000\u0000\u0000\u0189\u0594"+ - "\u0001\u0000\u0000\u0000\u018b\u059a\u0001\u0000\u0000\u0000\u018d\u05a0"+ - "\u0001\u0000\u0000\u0000\u018f\u05a4\u0001\u0000\u0000\u0000\u0191\u05a8"+ - "\u0001\u0000\u0000\u0000\u0193\u05ac\u0001\u0000\u0000\u0000\u0195\u05b2"+ - "\u0001\u0000\u0000\u0000\u0197\u05b8\u0001\u0000\u0000\u0000\u0199\u05be"+ - "\u0001\u0000\u0000\u0000\u019b\u019c\u0007\u0000\u0000\u0000\u019c\u019d"+ - "\u0007\u0001\u0000\u0000\u019d\u019e\u0007\u0002\u0000\u0000\u019e\u019f"+ - "\u0007\u0002\u0000\u0000\u019f\u01a0\u0007\u0003\u0000\u0000\u01a0\u01a1"+ - "\u0007\u0004\u0000\u0000\u01a1\u01a2\u0007\u0005\u0000\u0000\u01a2\u01a3"+ - "\u0001\u0000\u0000\u0000\u01a3\u01a4\u0006\u0000\u0000\u0000\u01a4\u0010"+ - "\u0001\u0000\u0000\u0000\u01a5\u01a6\u0007\u0000\u0000\u0000\u01a6\u01a7"+ - "\u0007\u0006\u0000\u0000\u01a7\u01a8\u0007\u0007\u0000\u0000\u01a8\u01a9"+ - "\u0007\b\u0000\u0000\u01a9\u01aa\u0001\u0000\u0000\u0000\u01aa\u01ab\u0006"+ - "\u0001\u0001\u0000\u01ab\u0012\u0001\u0000\u0000\u0000\u01ac\u01ad\u0007"+ - "\u0003\u0000\u0000\u01ad\u01ae\u0007\t\u0000\u0000\u01ae\u01af\u0007\u0006"+ - "\u0000\u0000\u01af\u01b0\u0007\u0001\u0000\u0000\u01b0\u01b1\u0007\u0004"+ - "\u0000\u0000\u01b1\u01b2\u0007\n\u0000\u0000\u01b2\u01b3\u0001\u0000\u0000"+ - "\u0000\u01b3\u01b4\u0006\u0002\u0002\u0000\u01b4\u0014\u0001\u0000\u0000"+ - "\u0000\u01b5\u01b6\u0007\u0003\u0000\u0000\u01b6\u01b7\u0007\u000b\u0000"+ - "\u0000\u01b7\u01b8\u0007\f\u0000\u0000\u01b8\u01b9\u0007\r\u0000\u0000"+ - "\u01b9\u01ba\u0001\u0000\u0000\u0000\u01ba\u01bb\u0006\u0003\u0000\u0000"+ - "\u01bb\u0016\u0001\u0000\u0000\u0000\u01bc\u01bd\u0007\u0003\u0000\u0000"+ - "\u01bd\u01be\u0007\u000e\u0000\u0000\u01be\u01bf\u0007\b\u0000\u0000\u01bf"+ - "\u01c0\u0007\r\u0000\u0000\u01c0\u01c1\u0007\f\u0000\u0000\u01c1\u01c2"+ - "\u0007\u0001\u0000\u0000\u01c2\u01c3\u0007\t\u0000\u0000\u01c3\u01c4\u0001"+ - "\u0000\u0000\u0000\u01c4\u01c5\u0006\u0004\u0003\u0000\u01c5\u0018\u0001"+ - "\u0000\u0000\u0000\u01c6\u01c7\u0007\u000f\u0000\u0000\u01c7\u01c8\u0007"+ - "\u0006\u0000\u0000\u01c8\u01c9\u0007\u0007\u0000\u0000\u01c9\u01ca\u0007"+ - "\u0010\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cc\u0006"+ - "\u0005\u0004\u0000\u01cc\u001a\u0001\u0000\u0000\u0000\u01cd\u01ce\u0007"+ - "\u0011\u0000\u0000\u01ce\u01cf\u0007\u0006\u0000\u0000\u01cf\u01d0\u0007"+ - "\u0007\u0000\u0000\u01d0\u01d1\u0007\u0012\u0000\u0000\u01d1\u01d2\u0001"+ - "\u0000\u0000\u0000\u01d2\u01d3\u0006\u0006\u0000\u0000\u01d3\u001c\u0001"+ - "\u0000\u0000\u0000\u01d4\u01d5\u0007\u0012\u0000\u0000\u01d5\u01d6\u0007"+ - "\u0003\u0000\u0000\u01d6\u01d7\u0007\u0003\u0000\u0000\u01d7\u01d8\u0007"+ - "\b\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da\u0006\u0007"+ - "\u0001\u0000\u01da\u001e\u0001\u0000\u0000\u0000\u01db\u01dc\u0007\r\u0000"+ - "\u0000\u01dc\u01dd\u0007\u0001\u0000\u0000\u01dd\u01de\u0007\u0010\u0000"+ - "\u0000\u01de\u01df\u0007\u0001\u0000\u0000\u01df\u01e0\u0007\u0005\u0000"+ - "\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e1\u01e2\u0006\b\u0000\u0000"+ - "\u01e2 \u0001\u0000\u0000\u0000\u01e3\u01e4\u0007\u0010\u0000\u0000\u01e4"+ - "\u01e5\u0007\u000b\u0000\u0000\u01e5\u01e6\u0005_\u0000\u0000\u01e6\u01e7"+ - "\u0007\u0003\u0000\u0000\u01e7\u01e8\u0007\u000e\u0000\u0000\u01e8\u01e9"+ - "\u0007\b\u0000\u0000\u01e9\u01ea\u0007\f\u0000\u0000\u01ea\u01eb\u0007"+ - "\t\u0000\u0000\u01eb\u01ec\u0007\u0000\u0000\u0000\u01ec\u01ed\u0001\u0000"+ - "\u0000\u0000\u01ed\u01ee\u0006\t\u0005\u0000\u01ee\"\u0001\u0000\u0000"+ - "\u0000\u01ef\u01f0\u0007\u0006\u0000\u0000\u01f0\u01f1\u0007\u0003\u0000"+ - "\u0000\u01f1\u01f2\u0007\t\u0000\u0000\u01f2\u01f3\u0007\f\u0000\u0000"+ - "\u01f3\u01f4\u0007\u0010\u0000\u0000\u01f4\u01f5\u0007\u0003\u0000\u0000"+ - "\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0006\n\u0006\u0000\u01f7"+ - "$\u0001\u0000\u0000\u0000\u01f8\u01f9\u0007\u0006\u0000\u0000\u01f9\u01fa"+ - "\u0007\u0007\u0000\u0000\u01fa\u01fb\u0007\u0013\u0000\u0000\u01fb\u01fc"+ - "\u0001\u0000\u0000\u0000\u01fc\u01fd\u0006\u000b\u0000\u0000\u01fd&\u0001"+ - "\u0000\u0000\u0000\u01fe\u01ff\u0007\u0002\u0000\u0000\u01ff\u0200\u0007"+ - "\n\u0000\u0000\u0200\u0201\u0007\u0007\u0000\u0000\u0201\u0202\u0007\u0013"+ - "\u0000\u0000\u0202\u0203\u0001\u0000\u0000\u0000\u0203\u0204\u0006\f\u0007"+ - "\u0000\u0204(\u0001\u0000\u0000\u0000\u0205\u0206\u0007\u0002\u0000\u0000"+ - "\u0206\u0207\u0007\u0007\u0000\u0000\u0207\u0208\u0007\u0006\u0000\u0000"+ - "\u0208\u0209\u0007\u0005\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000"+ - "\u020a\u020b\u0006\r\u0000\u0000\u020b*\u0001\u0000\u0000\u0000\u020c"+ - "\u020d\u0007\u0002\u0000\u0000\u020d\u020e\u0007\u0005\u0000\u0000\u020e"+ - "\u020f\u0007\f\u0000\u0000\u020f\u0210\u0007\u0005\u0000\u0000\u0210\u0211"+ - "\u0007\u0002\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212\u0213"+ - "\u0006\u000e\u0000\u0000\u0213,\u0001\u0000\u0000\u0000\u0214\u0215\u0007"+ - "\u0013\u0000\u0000\u0215\u0216\u0007\n\u0000\u0000\u0216\u0217\u0007\u0003"+ - "\u0000\u0000\u0217\u0218\u0007\u0006\u0000\u0000\u0218\u0219\u0007\u0003"+ - "\u0000\u0000\u0219\u021a\u0001\u0000\u0000\u0000\u021a\u021b\u0006\u000f"+ - "\u0000\u0000\u021b.\u0001\u0000\u0000\u0000\u021c\u021d\u0004\u0010\u0000"+ - "\u0000\u021d\u021e\u0007\u0001\u0000\u0000\u021e\u021f\u0007\t\u0000\u0000"+ - "\u021f\u0220\u0007\r\u0000\u0000\u0220\u0221\u0007\u0001\u0000\u0000\u0221"+ - "\u0222\u0007\t\u0000\u0000\u0222\u0223\u0007\u0003\u0000\u0000\u0223\u0224"+ - "\u0007\u0002\u0000\u0000\u0224\u0225\u0007\u0005\u0000\u0000\u0225\u0226"+ - "\u0007\f\u0000\u0000\u0226\u0227\u0007\u0005\u0000\u0000\u0227\u0228\u0007"+ - "\u0002\u0000\u0000\u0228\u0229\u0001\u0000\u0000\u0000\u0229\u022a\u0006"+ - "\u0010\u0000\u0000\u022a0\u0001\u0000\u0000\u0000\u022b\u022c\u0004\u0011"+ - "\u0001\u0000\u022c\u022d\u0007\r\u0000\u0000\u022d\u022e\u0007\u0007\u0000"+ - "\u0000\u022e\u022f\u0007\u0007\u0000\u0000\u022f\u0230\u0007\u0012\u0000"+ - "\u0000\u0230\u0231\u0007\u0014\u0000\u0000\u0231\u0232\u0007\b\u0000\u0000"+ - "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0006\u0011\b\u0000\u0234"+ - "2\u0001\u0000\u0000\u0000\u0235\u0236\u0004\u0012\u0002\u0000\u0236\u0237"+ - "\u0007\u0010\u0000\u0000\u0237\u0238\u0007\f\u0000\u0000\u0238\u0239\u0007"+ - "\u0005\u0000\u0000\u0239\u023a\u0007\u0004\u0000\u0000\u023a\u023b\u0007"+ - "\n\u0000\u0000\u023b\u023c\u0001\u0000\u0000\u0000\u023c\u023d\u0006\u0012"+ - "\u0000\u0000\u023d4\u0001\u0000\u0000\u0000\u023e\u023f\u0004\u0013\u0003"+ - "\u0000\u023f\u0240\u0007\u0010\u0000\u0000\u0240\u0241\u0007\u0003\u0000"+ - "\u0000\u0241\u0242\u0007\u0005\u0000\u0000\u0242\u0243\u0007\u0006\u0000"+ - "\u0000\u0243\u0244\u0007\u0001\u0000\u0000\u0244\u0245\u0007\u0004\u0000"+ - "\u0000\u0245\u0246\u0007\u0002\u0000\u0000\u0246\u0247\u0001\u0000\u0000"+ - "\u0000\u0247\u0248\u0006\u0013\t\u0000\u02486\u0001\u0000\u0000\u0000"+ - "\u0249\u024b\b\u0015\u0000\u0000\u024a\u0249\u0001\u0000\u0000\u0000\u024b"+ - "\u024c\u0001\u0000\u0000\u0000\u024c\u024a\u0001\u0000\u0000\u0000\u024c"+ - "\u024d\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e"+ - "\u024f\u0006\u0014\u0000\u0000\u024f8\u0001\u0000\u0000\u0000\u0250\u0251"+ - "\u0005/\u0000\u0000\u0251\u0252\u0005/\u0000\u0000\u0252\u0256\u0001\u0000"+ - "\u0000\u0000\u0253\u0255\b\u0016\u0000\u0000\u0254\u0253\u0001\u0000\u0000"+ - "\u0000\u0255\u0258\u0001\u0000\u0000\u0000\u0256\u0254\u0001\u0000\u0000"+ - "\u0000\u0256\u0257\u0001\u0000\u0000\u0000\u0257\u025a\u0001\u0000\u0000"+ - "\u0000\u0258\u0256\u0001\u0000\u0000\u0000\u0259\u025b\u0005\r\u0000\u0000"+ - "\u025a\u0259\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000\u0000"+ - "\u025b\u025d\u0001\u0000\u0000\u0000\u025c\u025e\u0005\n\u0000\u0000\u025d"+ - "\u025c\u0001\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000\u025e"+ - "\u025f\u0001\u0000\u0000\u0000\u025f\u0260\u0006\u0015\n\u0000\u0260:"+ - "\u0001\u0000\u0000\u0000\u0261\u0262\u0005/\u0000\u0000\u0262\u0263\u0005"+ - "*\u0000\u0000\u0263\u0268\u0001\u0000\u0000\u0000\u0264\u0267\u0003;\u0016"+ - "\u0000\u0265\u0267\t\u0000\u0000\u0000\u0266\u0264\u0001\u0000\u0000\u0000"+ - "\u0266\u0265\u0001\u0000\u0000\u0000\u0267\u026a\u0001\u0000\u0000\u0000"+ - "\u0268\u0269\u0001\u0000\u0000\u0000\u0268\u0266\u0001\u0000\u0000\u0000"+ - "\u0269\u026b\u0001\u0000\u0000\u0000\u026a\u0268\u0001\u0000\u0000\u0000"+ - "\u026b\u026c\u0005*\u0000\u0000\u026c\u026d\u0005/\u0000\u0000\u026d\u026e"+ - "\u0001\u0000\u0000\u0000\u026e\u026f\u0006\u0016\n\u0000\u026f<\u0001"+ - "\u0000\u0000\u0000\u0270\u0272\u0007\u0017\u0000\u0000\u0271\u0270\u0001"+ - "\u0000\u0000\u0000\u0272\u0273\u0001\u0000\u0000\u0000\u0273\u0271\u0001"+ - "\u0000\u0000\u0000\u0273\u0274\u0001\u0000\u0000\u0000\u0274\u0275\u0001"+ - "\u0000\u0000\u0000\u0275\u0276\u0006\u0017\n\u0000\u0276>\u0001\u0000"+ - "\u0000\u0000\u0277\u0278\u0005|\u0000\u0000\u0278\u0279\u0001\u0000\u0000"+ - "\u0000\u0279\u027a\u0006\u0018\u000b\u0000\u027a@\u0001\u0000\u0000\u0000"+ - "\u027b\u027c\u0007\u0018\u0000\u0000\u027cB\u0001\u0000\u0000\u0000\u027d"+ - "\u027e\u0007\u0019\u0000\u0000\u027eD\u0001\u0000\u0000\u0000\u027f\u0280"+ - "\u0005\\\u0000\u0000\u0280\u0281\u0007\u001a\u0000\u0000\u0281F\u0001"+ - "\u0000\u0000\u0000\u0282\u0283\b\u001b\u0000\u0000\u0283H\u0001\u0000"+ - "\u0000\u0000\u0284\u0286\u0007\u0003\u0000\u0000\u0285\u0287\u0007\u001c"+ - "\u0000\u0000\u0286\u0285\u0001\u0000\u0000\u0000\u0286\u0287\u0001\u0000"+ - "\u0000\u0000\u0287\u0289\u0001\u0000\u0000\u0000\u0288\u028a\u0003A\u0019"+ - "\u0000\u0289\u0288\u0001\u0000\u0000\u0000\u028a\u028b\u0001\u0000\u0000"+ - "\u0000\u028b\u0289\u0001\u0000\u0000\u0000\u028b\u028c\u0001\u0000\u0000"+ - "\u0000\u028cJ\u0001\u0000\u0000\u0000\u028d\u028e\u0005@\u0000\u0000\u028e"+ - "L\u0001\u0000\u0000\u0000\u028f\u0290\u0005`\u0000\u0000\u0290N\u0001"+ - "\u0000\u0000\u0000\u0291\u0295\b\u001d\u0000\u0000\u0292\u0293\u0005`"+ - "\u0000\u0000\u0293\u0295\u0005`\u0000\u0000\u0294\u0291\u0001\u0000\u0000"+ - "\u0000\u0294\u0292\u0001\u0000\u0000\u0000\u0295P\u0001\u0000\u0000\u0000"+ - "\u0296\u0297\u0005_\u0000\u0000\u0297R\u0001\u0000\u0000\u0000\u0298\u029c"+ - "\u0003C\u001a\u0000\u0299\u029c\u0003A\u0019\u0000\u029a\u029c\u0003Q"+ - "!\u0000\u029b\u0298\u0001\u0000\u0000\u0000\u029b\u0299\u0001\u0000\u0000"+ - "\u0000\u029b\u029a\u0001\u0000\u0000\u0000\u029cT\u0001\u0000\u0000\u0000"+ - "\u029d\u02a2\u0005\"\u0000\u0000\u029e\u02a1\u0003E\u001b\u0000\u029f"+ - "\u02a1\u0003G\u001c\u0000\u02a0\u029e\u0001\u0000\u0000\u0000\u02a0\u029f"+ - "\u0001\u0000\u0000\u0000\u02a1\u02a4\u0001\u0000\u0000\u0000\u02a2\u02a0"+ - "\u0001\u0000\u0000\u0000\u02a2\u02a3\u0001\u0000\u0000\u0000\u02a3\u02a5"+ - "\u0001\u0000\u0000\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5\u02bb"+ - "\u0005\"\u0000\u0000\u02a6\u02a7\u0005\"\u0000\u0000\u02a7\u02a8\u0005"+ - "\"\u0000\u0000\u02a8\u02a9\u0005\"\u0000\u0000\u02a9\u02ad\u0001\u0000"+ - "\u0000\u0000\u02aa\u02ac\b\u0016\u0000\u0000\u02ab\u02aa\u0001\u0000\u0000"+ - "\u0000\u02ac\u02af\u0001\u0000\u0000\u0000\u02ad\u02ae\u0001\u0000\u0000"+ - "\u0000\u02ad\u02ab\u0001\u0000\u0000\u0000\u02ae\u02b0\u0001\u0000\u0000"+ - "\u0000\u02af\u02ad\u0001\u0000\u0000\u0000\u02b0\u02b1\u0005\"\u0000\u0000"+ - "\u02b1\u02b2\u0005\"\u0000\u0000\u02b2\u02b3\u0005\"\u0000\u0000\u02b3"+ - "\u02b5\u0001\u0000\u0000\u0000\u02b4\u02b6\u0005\"\u0000\u0000\u02b5\u02b4"+ - "\u0001\u0000\u0000\u0000\u02b5\u02b6\u0001\u0000\u0000\u0000\u02b6\u02b8"+ - "\u0001\u0000\u0000\u0000\u02b7\u02b9\u0005\"\u0000\u0000\u02b8\u02b7\u0001"+ - "\u0000\u0000\u0000\u02b8\u02b9\u0001\u0000\u0000\u0000\u02b9\u02bb\u0001"+ - "\u0000\u0000\u0000\u02ba\u029d\u0001\u0000\u0000\u0000\u02ba\u02a6\u0001"+ - "\u0000\u0000\u0000\u02bbV\u0001\u0000\u0000\u0000\u02bc\u02be\u0003A\u0019"+ - "\u0000\u02bd\u02bc\u0001\u0000\u0000\u0000\u02be\u02bf\u0001\u0000\u0000"+ - "\u0000\u02bf\u02bd\u0001\u0000\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000"+ - "\u0000\u02c0X\u0001\u0000\u0000\u0000\u02c1\u02c3\u0003A\u0019\u0000\u02c2"+ - "\u02c1\u0001\u0000\u0000\u0000\u02c3\u02c4\u0001\u0000\u0000\u0000\u02c4"+ - "\u02c2\u0001\u0000\u0000\u0000\u02c4\u02c5\u0001\u0000\u0000\u0000\u02c5"+ - "\u02c6\u0001\u0000\u0000\u0000\u02c6\u02ca\u0003i-\u0000\u02c7\u02c9\u0003"+ - "A\u0019\u0000\u02c8\u02c7\u0001\u0000\u0000\u0000\u02c9\u02cc\u0001\u0000"+ - "\u0000\u0000\u02ca\u02c8\u0001\u0000\u0000\u0000\u02ca\u02cb\u0001\u0000"+ - "\u0000\u0000\u02cb\u02ec\u0001\u0000\u0000\u0000\u02cc\u02ca\u0001\u0000"+ - "\u0000\u0000\u02cd\u02cf\u0003i-\u0000\u02ce\u02d0\u0003A\u0019\u0000"+ - "\u02cf\u02ce\u0001\u0000\u0000\u0000\u02d0\u02d1\u0001\u0000\u0000\u0000"+ - "\u02d1\u02cf\u0001\u0000\u0000\u0000\u02d1\u02d2\u0001\u0000\u0000\u0000"+ - "\u02d2\u02ec\u0001\u0000\u0000\u0000\u02d3\u02d5\u0003A\u0019\u0000\u02d4"+ - "\u02d3\u0001\u0000\u0000\u0000\u02d5\u02d6\u0001\u0000\u0000\u0000\u02d6"+ - "\u02d4\u0001\u0000\u0000\u0000\u02d6\u02d7\u0001\u0000\u0000\u0000\u02d7"+ - "\u02df\u0001\u0000\u0000\u0000\u02d8\u02dc\u0003i-\u0000\u02d9\u02db\u0003"+ - "A\u0019\u0000\u02da\u02d9\u0001\u0000\u0000\u0000\u02db\u02de\u0001\u0000"+ - "\u0000\u0000\u02dc\u02da\u0001\u0000\u0000\u0000\u02dc\u02dd\u0001\u0000"+ - "\u0000\u0000\u02dd\u02e0\u0001\u0000\u0000\u0000\u02de\u02dc\u0001\u0000"+ - "\u0000\u0000\u02df\u02d8\u0001\u0000\u0000\u0000\u02df\u02e0\u0001\u0000"+ - "\u0000\u0000\u02e0\u02e1\u0001\u0000\u0000\u0000\u02e1\u02e2\u0003I\u001d"+ - "\u0000\u02e2\u02ec\u0001\u0000\u0000\u0000\u02e3\u02e5\u0003i-\u0000\u02e4"+ - "\u02e6\u0003A\u0019\u0000\u02e5\u02e4\u0001\u0000\u0000\u0000\u02e6\u02e7"+ - "\u0001\u0000\u0000\u0000\u02e7\u02e5\u0001\u0000\u0000\u0000\u02e7\u02e8"+ - "\u0001\u0000\u0000\u0000\u02e8\u02e9\u0001\u0000\u0000\u0000\u02e9\u02ea"+ - "\u0003I\u001d\u0000\u02ea\u02ec\u0001\u0000\u0000\u0000\u02eb\u02c2\u0001"+ - "\u0000\u0000\u0000\u02eb\u02cd\u0001\u0000\u0000\u0000\u02eb\u02d4\u0001"+ - "\u0000\u0000\u0000\u02eb\u02e3\u0001\u0000\u0000\u0000\u02ecZ\u0001\u0000"+ - "\u0000\u0000\u02ed\u02ee\u0007\u001e\u0000\u0000\u02ee\u02ef\u0007\u001f"+ - "\u0000\u0000\u02ef\\\u0001\u0000\u0000\u0000\u02f0\u02f1\u0007\f\u0000"+ - "\u0000\u02f1\u02f2\u0007\t\u0000\u0000\u02f2\u02f3\u0007\u0000\u0000\u0000"+ - "\u02f3^\u0001\u0000\u0000\u0000\u02f4\u02f5\u0007\f\u0000\u0000\u02f5"+ - "\u02f6\u0007\u0002\u0000\u0000\u02f6\u02f7\u0007\u0004\u0000\u0000\u02f7"+ - "`\u0001\u0000\u0000\u0000\u02f8\u02f9\u0005=\u0000\u0000\u02f9b\u0001"+ - "\u0000\u0000\u0000\u02fa\u02fb\u0005:\u0000\u0000\u02fb\u02fc\u0005:\u0000"+ - "\u0000\u02fcd\u0001\u0000\u0000\u0000\u02fd\u02fe\u0005,\u0000\u0000\u02fe"+ - "f\u0001\u0000\u0000\u0000\u02ff\u0300\u0007\u0000\u0000\u0000\u0300\u0301"+ - "\u0007\u0003\u0000\u0000\u0301\u0302\u0007\u0002\u0000\u0000\u0302\u0303"+ - "\u0007\u0004\u0000\u0000\u0303h\u0001\u0000\u0000\u0000\u0304\u0305\u0005"+ - ".\u0000\u0000\u0305j\u0001\u0000\u0000\u0000\u0306\u0307\u0007\u000f\u0000"+ - "\u0000\u0307\u0308\u0007\f\u0000\u0000\u0308\u0309\u0007\r\u0000\u0000"+ - "\u0309\u030a\u0007\u0002\u0000\u0000\u030a\u030b\u0007\u0003\u0000\u0000"+ - "\u030bl\u0001\u0000\u0000\u0000\u030c\u030d\u0007\u000f\u0000\u0000\u030d"+ - "\u030e\u0007\u0001\u0000\u0000\u030e\u030f\u0007\u0006\u0000\u0000\u030f"+ - "\u0310\u0007\u0002\u0000\u0000\u0310\u0311\u0007\u0005\u0000\u0000\u0311"+ - "n\u0001\u0000\u0000\u0000\u0312\u0313\u0007\u0001\u0000\u0000\u0313\u0314"+ - "\u0007\t\u0000\u0000\u0314p\u0001\u0000\u0000\u0000\u0315\u0316\u0007"+ - "\u0001\u0000\u0000\u0316\u0317\u0007\u0002\u0000\u0000\u0317r\u0001\u0000"+ - "\u0000\u0000\u0318\u0319\u0007\r\u0000\u0000\u0319\u031a\u0007\f\u0000"+ - "\u0000\u031a\u031b\u0007\u0002\u0000\u0000\u031b\u031c\u0007\u0005\u0000"+ - "\u0000\u031ct\u0001\u0000\u0000\u0000\u031d\u031e\u0007\r\u0000\u0000"+ - "\u031e\u031f\u0007\u0001\u0000\u0000\u031f\u0320\u0007\u0012\u0000\u0000"+ - "\u0320\u0321\u0007\u0003\u0000\u0000\u0321v\u0001\u0000\u0000\u0000\u0322"+ - "\u0323\u0005(\u0000\u0000\u0323x\u0001\u0000\u0000\u0000\u0324\u0325\u0007"+ - "\t\u0000\u0000\u0325\u0326\u0007\u0007\u0000\u0000\u0326\u0327\u0007\u0005"+ - "\u0000\u0000\u0327z\u0001\u0000\u0000\u0000\u0328\u0329\u0007\t\u0000"+ - "\u0000\u0329\u032a\u0007\u0014\u0000\u0000\u032a\u032b\u0007\r\u0000\u0000"+ - "\u032b\u032c\u0007\r\u0000\u0000\u032c|\u0001\u0000\u0000\u0000\u032d"+ - "\u032e\u0007\t\u0000\u0000\u032e\u032f\u0007\u0014\u0000\u0000\u032f\u0330"+ - "\u0007\r\u0000\u0000\u0330\u0331\u0007\r\u0000\u0000\u0331\u0332\u0007"+ - "\u0002\u0000\u0000\u0332~\u0001\u0000\u0000\u0000\u0333\u0334\u0007\u0007"+ - "\u0000\u0000\u0334\u0335\u0007\u0006\u0000\u0000\u0335\u0080\u0001\u0000"+ - "\u0000\u0000\u0336\u0337\u0005?\u0000\u0000\u0337\u0082\u0001\u0000\u0000"+ - "\u0000\u0338\u0339\u0007\u0006\u0000\u0000\u0339\u033a\u0007\r\u0000\u0000"+ - "\u033a\u033b\u0007\u0001\u0000\u0000\u033b\u033c\u0007\u0012\u0000\u0000"+ - "\u033c\u033d\u0007\u0003\u0000\u0000\u033d\u0084\u0001\u0000\u0000\u0000"+ - "\u033e\u033f\u0005)\u0000\u0000\u033f\u0086\u0001\u0000\u0000\u0000\u0340"+ - "\u0341\u0007\u0005\u0000\u0000\u0341\u0342\u0007\u0006\u0000\u0000\u0342"+ - "\u0343\u0007\u0014\u0000\u0000\u0343\u0344\u0007\u0003\u0000\u0000\u0344"+ - "\u0088\u0001\u0000\u0000\u0000\u0345\u0346\u0005=\u0000\u0000\u0346\u0347"+ - "\u0005=\u0000\u0000\u0347\u008a\u0001\u0000\u0000\u0000\u0348\u0349\u0005"+ - "=\u0000\u0000\u0349\u034a\u0005~\u0000\u0000\u034a\u008c\u0001\u0000\u0000"+ - "\u0000\u034b\u034c\u0005!\u0000\u0000\u034c\u034d\u0005=\u0000\u0000\u034d"+ - "\u008e\u0001\u0000\u0000\u0000\u034e\u034f\u0005<\u0000\u0000\u034f\u0090"+ - "\u0001\u0000\u0000\u0000\u0350\u0351\u0005<\u0000\u0000\u0351\u0352\u0005"+ - "=\u0000\u0000\u0352\u0092\u0001\u0000\u0000\u0000\u0353\u0354\u0005>\u0000"+ - "\u0000\u0354\u0094\u0001\u0000\u0000\u0000\u0355\u0356\u0005>\u0000\u0000"+ - "\u0356\u0357\u0005=\u0000\u0000\u0357\u0096\u0001\u0000\u0000\u0000\u0358"+ - "\u0359\u0005+\u0000\u0000\u0359\u0098\u0001\u0000\u0000\u0000\u035a\u035b"+ - "\u0005-\u0000\u0000\u035b\u009a\u0001\u0000\u0000\u0000\u035c\u035d\u0005"+ - "*\u0000\u0000\u035d\u009c\u0001\u0000\u0000\u0000\u035e\u035f\u0005/\u0000"+ - "\u0000\u035f\u009e\u0001\u0000\u0000\u0000\u0360\u0361\u0005%\u0000\u0000"+ - "\u0361\u00a0\u0001\u0000\u0000\u0000\u0362\u0363\u0004I\u0004\u0000\u0363"+ - "\u0364\u00033\u0012\u0000\u0364\u0365\u0001\u0000\u0000\u0000\u0365\u0366"+ - "\u0006I\f\u0000\u0366\u00a2\u0001\u0000\u0000\u0000\u0367\u036a\u0003"+ - "\u00819\u0000\u0368\u036b\u0003C\u001a\u0000\u0369\u036b\u0003Q!\u0000"+ - "\u036a\u0368\u0001\u0000\u0000\u0000\u036a\u0369\u0001\u0000\u0000\u0000"+ - "\u036b\u036f\u0001\u0000\u0000\u0000\u036c\u036e\u0003S\"\u0000\u036d"+ - "\u036c\u0001\u0000\u0000\u0000\u036e\u0371\u0001\u0000\u0000\u0000\u036f"+ - "\u036d\u0001\u0000\u0000\u0000\u036f\u0370\u0001\u0000\u0000\u0000\u0370"+ - "\u0379\u0001\u0000\u0000\u0000\u0371\u036f\u0001\u0000\u0000\u0000\u0372"+ - "\u0374\u0003\u00819\u0000\u0373\u0375\u0003A\u0019\u0000\u0374\u0373\u0001"+ - "\u0000\u0000\u0000\u0375\u0376\u0001\u0000\u0000\u0000\u0376\u0374\u0001"+ - "\u0000\u0000\u0000\u0376\u0377\u0001\u0000\u0000\u0000\u0377\u0379\u0001"+ - "\u0000\u0000\u0000\u0378\u0367\u0001\u0000\u0000\u0000\u0378\u0372\u0001"+ - "\u0000\u0000\u0000\u0379\u00a4\u0001\u0000\u0000\u0000\u037a\u037b\u0005"+ - "[\u0000\u0000\u037b\u037c\u0001\u0000\u0000\u0000\u037c\u037d\u0006K\u0000"+ - "\u0000\u037d\u037e\u0006K\u0000\u0000\u037e\u00a6\u0001\u0000\u0000\u0000"+ - "\u037f\u0380\u0005]\u0000\u0000\u0380\u0381\u0001\u0000\u0000\u0000\u0381"+ - "\u0382\u0006L\u000b\u0000\u0382\u0383\u0006L\u000b\u0000\u0383\u00a8\u0001"+ - "\u0000\u0000\u0000\u0384\u0388\u0003C\u001a\u0000\u0385\u0387\u0003S\""+ - "\u0000\u0386\u0385\u0001\u0000\u0000\u0000\u0387\u038a\u0001\u0000\u0000"+ - "\u0000\u0388\u0386\u0001\u0000\u0000\u0000\u0388\u0389\u0001\u0000\u0000"+ - "\u0000\u0389\u0395\u0001\u0000\u0000\u0000\u038a\u0388\u0001\u0000\u0000"+ - "\u0000\u038b\u038e\u0003Q!\u0000\u038c\u038e\u0003K\u001e\u0000\u038d"+ - "\u038b\u0001\u0000\u0000\u0000\u038d\u038c\u0001\u0000\u0000\u0000\u038e"+ - "\u0390\u0001\u0000\u0000\u0000\u038f\u0391\u0003S\"\u0000\u0390\u038f"+ - "\u0001\u0000\u0000\u0000\u0391\u0392\u0001\u0000\u0000\u0000\u0392\u0390"+ - "\u0001\u0000\u0000\u0000\u0392\u0393\u0001\u0000\u0000\u0000\u0393\u0395"+ - "\u0001\u0000\u0000\u0000\u0394\u0384\u0001\u0000\u0000\u0000\u0394\u038d"+ - "\u0001\u0000\u0000\u0000\u0395\u00aa\u0001\u0000\u0000\u0000\u0396\u0398"+ - "\u0003M\u001f\u0000\u0397\u0399\u0003O \u0000\u0398\u0397\u0001\u0000"+ - "\u0000\u0000\u0399\u039a\u0001\u0000\u0000\u0000\u039a\u0398\u0001\u0000"+ - "\u0000\u0000\u039a\u039b\u0001\u0000\u0000\u0000\u039b\u039c\u0001\u0000"+ - "\u0000\u0000\u039c\u039d\u0003M\u001f\u0000\u039d\u00ac\u0001\u0000\u0000"+ - "\u0000\u039e\u039f\u0003\u00abN\u0000\u039f\u00ae\u0001\u0000\u0000\u0000"+ - "\u03a0\u03a1\u00039\u0015\u0000\u03a1\u03a2\u0001\u0000\u0000\u0000\u03a2"+ - "\u03a3\u0006P\n\u0000\u03a3\u00b0\u0001\u0000\u0000\u0000\u03a4\u03a5"+ - "\u0003;\u0016\u0000\u03a5\u03a6\u0001\u0000\u0000\u0000\u03a6\u03a7\u0006"+ - "Q\n\u0000\u03a7\u00b2\u0001\u0000\u0000\u0000\u03a8\u03a9\u0003=\u0017"+ - "\u0000\u03a9\u03aa\u0001\u0000\u0000\u0000\u03aa\u03ab\u0006R\n\u0000"+ - "\u03ab\u00b4\u0001\u0000\u0000\u0000\u03ac\u03ad\u0003\u00a5K\u0000\u03ad"+ - "\u03ae\u0001\u0000\u0000\u0000\u03ae\u03af\u0006S\r\u0000\u03af\u03b0"+ - "\u0006S\u000e\u0000\u03b0\u00b6\u0001\u0000\u0000\u0000\u03b1\u03b2\u0003"+ - "?\u0018\u0000\u03b2\u03b3\u0001\u0000\u0000\u0000\u03b3\u03b4\u0006T\u000f"+ - "\u0000\u03b4\u03b5\u0006T\u000b\u0000\u03b5\u00b8\u0001\u0000\u0000\u0000"+ - "\u03b6\u03b7\u0003=\u0017\u0000\u03b7\u03b8\u0001\u0000\u0000\u0000\u03b8"+ - "\u03b9\u0006U\n\u0000\u03b9\u00ba\u0001\u0000\u0000\u0000\u03ba\u03bb"+ - "\u00039\u0015\u0000\u03bb\u03bc\u0001\u0000\u0000\u0000\u03bc\u03bd\u0006"+ - "V\n\u0000\u03bd\u00bc\u0001\u0000\u0000\u0000\u03be\u03bf\u0003;\u0016"+ - "\u0000\u03bf\u03c0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006W\n\u0000"+ - "\u03c1\u00be\u0001\u0000\u0000\u0000\u03c2\u03c3\u0003?\u0018\u0000\u03c3"+ - "\u03c4\u0001\u0000\u0000\u0000\u03c4\u03c5\u0006X\u000f\u0000\u03c5\u03c6"+ - "\u0006X\u000b\u0000\u03c6\u00c0\u0001\u0000\u0000\u0000\u03c7\u03c8\u0003"+ - "\u00a5K\u0000\u03c8\u03c9\u0001\u0000\u0000\u0000\u03c9\u03ca\u0006Y\r"+ - "\u0000\u03ca\u00c2\u0001\u0000\u0000\u0000\u03cb\u03cc\u0003\u00a7L\u0000"+ - "\u03cc\u03cd\u0001\u0000\u0000\u0000\u03cd\u03ce\u0006Z\u0010\u0000\u03ce"+ - "\u00c4\u0001\u0000\u0000\u0000\u03cf\u03d0\u0003\u0151\u00a1\u0000\u03d0"+ - "\u03d1\u0001\u0000\u0000\u0000\u03d1\u03d2\u0006[\u0011\u0000\u03d2\u00c6"+ - "\u0001\u0000\u0000\u0000\u03d3\u03d4\u0003e+\u0000\u03d4\u03d5\u0001\u0000"+ - "\u0000\u0000\u03d5\u03d6\u0006\\\u0012\u0000\u03d6\u00c8\u0001\u0000\u0000"+ - "\u0000\u03d7\u03d8\u0003a)\u0000\u03d8\u03d9\u0001\u0000\u0000\u0000\u03d9"+ - "\u03da\u0006]\u0013\u0000\u03da\u00ca\u0001\u0000\u0000\u0000\u03db\u03dc"+ - "\u0007\u0010\u0000\u0000\u03dc\u03dd\u0007\u0003\u0000\u0000\u03dd\u03de"+ - "\u0007\u0005\u0000\u0000\u03de\u03df\u0007\f\u0000\u0000\u03df\u03e0\u0007"+ - "\u0000\u0000\u0000\u03e0\u03e1\u0007\f\u0000\u0000\u03e1\u03e2\u0007\u0005"+ - "\u0000\u0000\u03e2\u03e3\u0007\f\u0000\u0000\u03e3\u00cc\u0001\u0000\u0000"+ - "\u0000\u03e4\u03e8\b \u0000\u0000\u03e5\u03e6\u0005/\u0000\u0000\u03e6"+ - "\u03e8\b!\u0000\u0000\u03e7\u03e4\u0001\u0000\u0000\u0000\u03e7\u03e5"+ - "\u0001\u0000\u0000\u0000\u03e8\u00ce\u0001\u0000\u0000\u0000\u03e9\u03eb"+ - "\u0003\u00cd_\u0000\u03ea\u03e9\u0001\u0000\u0000\u0000\u03eb\u03ec\u0001"+ - "\u0000\u0000\u0000\u03ec\u03ea\u0001\u0000\u0000\u0000\u03ec\u03ed\u0001"+ - "\u0000\u0000\u0000\u03ed\u00d0\u0001\u0000\u0000\u0000\u03ee\u03ef\u0003"+ - "\u00cf`\u0000\u03ef\u03f0\u0001\u0000\u0000\u0000\u03f0\u03f1\u0006a\u0014"+ - "\u0000\u03f1\u00d2\u0001\u0000\u0000\u0000\u03f2\u03f3\u0003U#\u0000\u03f3"+ - "\u03f4\u0001\u0000\u0000\u0000\u03f4\u03f5\u0006b\u0015\u0000\u03f5\u00d4"+ - "\u0001\u0000\u0000\u0000\u03f6\u03f7\u00039\u0015\u0000\u03f7\u03f8\u0001"+ - "\u0000\u0000\u0000\u03f8\u03f9\u0006c\n\u0000\u03f9\u00d6\u0001\u0000"+ - "\u0000\u0000\u03fa\u03fb\u0003;\u0016\u0000\u03fb\u03fc\u0001\u0000\u0000"+ - "\u0000\u03fc\u03fd\u0006d\n\u0000\u03fd\u00d8\u0001\u0000\u0000\u0000"+ - "\u03fe\u03ff\u0003=\u0017\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400"+ - "\u0401\u0006e\n\u0000\u0401\u00da\u0001\u0000\u0000\u0000\u0402\u0403"+ - "\u0003?\u0018\u0000\u0403\u0404\u0001\u0000\u0000\u0000\u0404\u0405\u0006"+ - "f\u000f\u0000\u0405\u0406\u0006f\u000b\u0000\u0406\u00dc\u0001\u0000\u0000"+ - "\u0000\u0407\u0408\u0003i-\u0000\u0408\u0409\u0001\u0000\u0000\u0000\u0409"+ - "\u040a\u0006g\u0016\u0000\u040a\u00de\u0001\u0000\u0000\u0000\u040b\u040c"+ - "\u0003e+\u0000\u040c\u040d\u0001\u0000\u0000\u0000\u040d\u040e\u0006h"+ - "\u0012\u0000\u040e\u00e0\u0001\u0000\u0000\u0000\u040f\u0410\u0003\u0081"+ - "9\u0000\u0410\u0411\u0001\u0000\u0000\u0000\u0411\u0412\u0006i\u0017\u0000"+ - "\u0412\u00e2\u0001\u0000\u0000\u0000\u0413\u0414\u0003\u00a3J\u0000\u0414"+ - "\u0415\u0001\u0000\u0000\u0000\u0415\u0416\u0006j\u0018\u0000\u0416\u00e4"+ - "\u0001\u0000\u0000\u0000\u0417\u041c\u0003C\u001a\u0000\u0418\u041c\u0003"+ - "A\u0019\u0000\u0419\u041c\u0003Q!\u0000\u041a\u041c\u0003\u009bF\u0000"+ - "\u041b\u0417\u0001\u0000\u0000\u0000\u041b\u0418\u0001\u0000\u0000\u0000"+ - "\u041b\u0419\u0001\u0000\u0000\u0000\u041b\u041a\u0001\u0000\u0000\u0000"+ - "\u041c\u00e6\u0001\u0000\u0000\u0000\u041d\u0420\u0003C\u001a\u0000\u041e"+ - "\u0420\u0003\u009bF\u0000\u041f\u041d\u0001\u0000\u0000\u0000\u041f\u041e"+ - "\u0001\u0000\u0000\u0000\u0420\u0424\u0001\u0000\u0000\u0000\u0421\u0423"+ - "\u0003\u00e5k\u0000\u0422\u0421\u0001\u0000\u0000\u0000\u0423\u0426\u0001"+ - "\u0000\u0000\u0000\u0424\u0422\u0001\u0000\u0000\u0000\u0424\u0425\u0001"+ - "\u0000\u0000\u0000\u0425\u0431\u0001\u0000\u0000\u0000\u0426\u0424\u0001"+ - "\u0000\u0000\u0000\u0427\u042a\u0003Q!\u0000\u0428\u042a\u0003K\u001e"+ - "\u0000\u0429\u0427\u0001\u0000\u0000\u0000\u0429\u0428\u0001\u0000\u0000"+ - "\u0000\u042a\u042c\u0001\u0000\u0000\u0000\u042b\u042d\u0003\u00e5k\u0000"+ - "\u042c\u042b\u0001\u0000\u0000\u0000\u042d\u042e\u0001\u0000\u0000\u0000"+ - "\u042e\u042c\u0001\u0000\u0000\u0000\u042e\u042f\u0001\u0000\u0000\u0000"+ - "\u042f\u0431\u0001\u0000\u0000\u0000\u0430\u041f\u0001\u0000\u0000\u0000"+ - "\u0430\u0429\u0001\u0000\u0000\u0000\u0431\u00e8\u0001\u0000\u0000\u0000"+ - "\u0432\u0435\u0003\u00e7l\u0000\u0433\u0435\u0003\u00abN\u0000\u0434\u0432"+ - "\u0001\u0000\u0000\u0000\u0434\u0433\u0001\u0000\u0000\u0000\u0435\u0436"+ - "\u0001\u0000\u0000\u0000\u0436\u0434\u0001\u0000\u0000\u0000\u0436\u0437"+ - "\u0001\u0000\u0000\u0000\u0437\u00ea\u0001\u0000\u0000\u0000\u0438\u0439"+ - "\u00039\u0015\u0000\u0439\u043a\u0001\u0000\u0000\u0000\u043a\u043b\u0006"+ - "n\n\u0000\u043b\u00ec\u0001\u0000\u0000\u0000\u043c\u043d\u0003;\u0016"+ - "\u0000\u043d\u043e\u0001\u0000\u0000\u0000\u043e\u043f\u0006o\n\u0000"+ - "\u043f\u00ee\u0001\u0000\u0000\u0000\u0440\u0441\u0003=\u0017\u0000\u0441"+ - "\u0442\u0001\u0000\u0000\u0000\u0442\u0443\u0006p\n\u0000\u0443\u00f0"+ - "\u0001\u0000\u0000\u0000\u0444\u0445\u0003?\u0018\u0000\u0445\u0446\u0001"+ - "\u0000\u0000\u0000\u0446\u0447\u0006q\u000f\u0000\u0447\u0448\u0006q\u000b"+ - "\u0000\u0448\u00f2\u0001\u0000\u0000\u0000\u0449\u044a\u0003a)\u0000\u044a"+ - "\u044b\u0001\u0000\u0000\u0000\u044b\u044c\u0006r\u0013\u0000\u044c\u00f4"+ - "\u0001\u0000\u0000\u0000\u044d\u044e\u0003e+\u0000\u044e\u044f\u0001\u0000"+ - "\u0000\u0000\u044f\u0450\u0006s\u0012\u0000\u0450\u00f6\u0001\u0000\u0000"+ - "\u0000\u0451\u0452\u0003i-\u0000\u0452\u0453\u0001\u0000\u0000\u0000\u0453"+ - "\u0454\u0006t\u0016\u0000\u0454\u00f8\u0001\u0000\u0000\u0000\u0455\u0456"+ - "\u0003\u00819\u0000\u0456\u0457\u0001\u0000\u0000\u0000\u0457\u0458\u0006"+ - "u\u0017\u0000\u0458\u00fa\u0001\u0000\u0000\u0000\u0459\u045a\u0003\u00a3"+ - "J\u0000\u045a\u045b\u0001\u0000\u0000\u0000\u045b\u045c\u0006v\u0018\u0000"+ - "\u045c\u00fc\u0001\u0000\u0000\u0000\u045d\u045e\u0007\f\u0000\u0000\u045e"+ - "\u045f\u0007\u0002\u0000\u0000\u045f\u00fe\u0001\u0000\u0000\u0000\u0460"+ - "\u0461\u0003\u00e9m\u0000\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463"+ - "\u0006x\u0019\u0000\u0463\u0100\u0001\u0000\u0000\u0000\u0464\u0465\u0003"+ - "9\u0015\u0000\u0465\u0466\u0001\u0000\u0000\u0000\u0466\u0467\u0006y\n"+ - "\u0000\u0467\u0102\u0001\u0000\u0000\u0000\u0468\u0469\u0003;\u0016\u0000"+ - "\u0469\u046a\u0001\u0000\u0000\u0000\u046a\u046b\u0006z\n\u0000\u046b"+ - "\u0104\u0001\u0000\u0000\u0000\u046c\u046d\u0003=\u0017\u0000\u046d\u046e"+ - "\u0001\u0000\u0000\u0000\u046e\u046f\u0006{\n\u0000\u046f\u0106\u0001"+ - "\u0000\u0000\u0000\u0470\u0471\u0003?\u0018\u0000\u0471\u0472\u0001\u0000"+ - "\u0000\u0000\u0472\u0473\u0006|\u000f\u0000\u0473\u0474\u0006|\u000b\u0000"+ - "\u0474\u0108\u0001\u0000\u0000\u0000\u0475\u0476\u0003\u00a5K\u0000\u0476"+ - "\u0477\u0001\u0000\u0000\u0000\u0477\u0478\u0006}\r\u0000\u0478\u0479"+ - "\u0006}\u001a\u0000\u0479\u010a\u0001\u0000\u0000\u0000\u047a\u047b\u0007"+ - "\u0007\u0000\u0000\u047b\u047c\u0007\t\u0000\u0000\u047c\u047d\u0001\u0000"+ - "\u0000\u0000\u047d\u047e\u0006~\u001b\u0000\u047e\u010c\u0001\u0000\u0000"+ - "\u0000\u047f\u0480\u0007\u0013\u0000\u0000\u0480\u0481\u0007\u0001\u0000"+ - "\u0000\u0481\u0482\u0007\u0005\u0000\u0000\u0482\u0483\u0007\n\u0000\u0000"+ - "\u0483\u0484\u0001\u0000\u0000\u0000\u0484\u0485\u0006\u007f\u001b\u0000"+ - "\u0485\u010e\u0001\u0000\u0000\u0000\u0486\u0487\b\"\u0000\u0000\u0487"+ - "\u0110\u0001\u0000\u0000\u0000\u0488\u048a\u0003\u010f\u0080\u0000\u0489"+ - "\u0488\u0001\u0000\u0000\u0000\u048a\u048b\u0001\u0000\u0000\u0000\u048b"+ - "\u0489\u0001\u0000\u0000\u0000\u048b\u048c\u0001\u0000\u0000\u0000\u048c"+ - "\u048d\u0001\u0000\u0000\u0000\u048d\u048e\u0003\u0151\u00a1\u0000\u048e"+ - "\u0490\u0001\u0000\u0000\u0000\u048f\u0489\u0001\u0000\u0000\u0000\u048f"+ - "\u0490\u0001\u0000\u0000\u0000\u0490\u0492\u0001\u0000\u0000\u0000\u0491"+ - "\u0493\u0003\u010f\u0080\u0000\u0492\u0491\u0001\u0000\u0000\u0000\u0493"+ - "\u0494\u0001\u0000\u0000\u0000\u0494\u0492\u0001\u0000\u0000\u0000\u0494"+ - "\u0495\u0001\u0000\u0000\u0000\u0495\u0112\u0001\u0000\u0000\u0000\u0496"+ - "\u0497\u0003\u0111\u0081\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498"+ - "\u0499\u0006\u0082\u001c\u0000\u0499\u0114\u0001\u0000\u0000\u0000\u049a"+ - "\u049b\u00039\u0015\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d"+ - "\u0006\u0083\n\u0000\u049d\u0116\u0001\u0000\u0000\u0000\u049e\u049f\u0003"+ - ";\u0016\u0000\u049f\u04a0\u0001\u0000\u0000\u0000\u04a0\u04a1\u0006\u0084"+ - "\n\u0000\u04a1\u0118\u0001\u0000\u0000\u0000\u04a2\u04a3\u0003=\u0017"+ - "\u0000\u04a3\u04a4\u0001\u0000\u0000\u0000\u04a4\u04a5\u0006\u0085\n\u0000"+ - "\u04a5\u011a\u0001\u0000\u0000\u0000\u04a6\u04a7\u0003?\u0018\u0000\u04a7"+ - "\u04a8\u0001\u0000\u0000\u0000\u04a8\u04a9\u0006\u0086\u000f\u0000\u04a9"+ - "\u04aa\u0006\u0086\u000b\u0000\u04aa\u04ab\u0006\u0086\u000b\u0000\u04ab"+ - "\u011c\u0001\u0000\u0000\u0000\u04ac\u04ad\u0003a)\u0000\u04ad\u04ae\u0001"+ - "\u0000\u0000\u0000\u04ae\u04af\u0006\u0087\u0013\u0000\u04af\u011e\u0001"+ - "\u0000\u0000\u0000\u04b0\u04b1\u0003e+\u0000\u04b1\u04b2\u0001\u0000\u0000"+ - "\u0000\u04b2\u04b3\u0006\u0088\u0012\u0000\u04b3\u0120\u0001\u0000\u0000"+ - "\u0000\u04b4\u04b5\u0003i-\u0000\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6"+ - "\u04b7\u0006\u0089\u0016\u0000\u04b7\u0122\u0001\u0000\u0000\u0000\u04b8"+ - "\u04b9\u0003\u010d\u007f\u0000\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba"+ - "\u04bb\u0006\u008a\u001d\u0000\u04bb\u0124\u0001\u0000\u0000\u0000\u04bc"+ - "\u04bd\u0003\u00e9m\u0000\u04bd\u04be\u0001\u0000\u0000\u0000\u04be\u04bf"+ - "\u0006\u008b\u0019\u0000\u04bf\u0126\u0001\u0000\u0000\u0000\u04c0\u04c1"+ - "\u0003\u00adO\u0000\u04c1\u04c2\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006"+ - "\u008c\u001e\u0000\u04c3\u0128\u0001\u0000\u0000\u0000\u04c4\u04c5\u0003"+ - "\u00819\u0000\u04c5\u04c6\u0001\u0000\u0000\u0000\u04c6\u04c7\u0006\u008d"+ - "\u0017\u0000\u04c7\u012a\u0001\u0000\u0000\u0000\u04c8\u04c9\u0003\u00a3"+ - "J\u0000\u04c9\u04ca\u0001\u0000\u0000\u0000\u04ca\u04cb\u0006\u008e\u0018"+ - "\u0000\u04cb\u012c\u0001\u0000\u0000\u0000\u04cc\u04cd\u00039\u0015\u0000"+ - "\u04cd\u04ce\u0001\u0000\u0000\u0000\u04ce\u04cf\u0006\u008f\n\u0000\u04cf"+ - "\u012e\u0001\u0000\u0000\u0000\u04d0\u04d1\u0003;\u0016\u0000\u04d1\u04d2"+ - "\u0001\u0000\u0000\u0000\u04d2\u04d3\u0006\u0090\n\u0000\u04d3\u0130\u0001"+ - "\u0000\u0000\u0000\u04d4\u04d5\u0003=\u0017\u0000\u04d5\u04d6\u0001\u0000"+ - "\u0000\u0000\u04d6\u04d7\u0006\u0091\n\u0000\u04d7\u0132\u0001\u0000\u0000"+ - "\u0000\u04d8\u04d9\u0003?\u0018\u0000\u04d9\u04da\u0001\u0000\u0000\u0000"+ - "\u04da\u04db\u0006\u0092\u000f\u0000\u04db\u04dc\u0006\u0092\u000b\u0000"+ - "\u04dc\u0134\u0001\u0000\u0000\u0000\u04dd\u04de\u0003i-\u0000\u04de\u04df"+ - "\u0001\u0000\u0000\u0000\u04df\u04e0\u0006\u0093\u0016\u0000\u04e0\u0136"+ - "\u0001\u0000\u0000\u0000\u04e1\u04e2\u0003\u00819\u0000\u04e2\u04e3\u0001"+ - "\u0000\u0000\u0000\u04e3\u04e4\u0006\u0094\u0017\u0000\u04e4\u0138\u0001"+ - "\u0000\u0000\u0000\u04e5\u04e6\u0003\u00a3J\u0000\u04e6\u04e7\u0001\u0000"+ - "\u0000\u0000\u04e7\u04e8\u0006\u0095\u0018\u0000\u04e8\u013a\u0001\u0000"+ - "\u0000\u0000\u04e9\u04ea\u0003\u00adO\u0000\u04ea\u04eb\u0001\u0000\u0000"+ - "\u0000\u04eb\u04ec\u0006\u0096\u001e\u0000\u04ec\u013c\u0001\u0000\u0000"+ - "\u0000\u04ed\u04ee\u0003\u00a9M\u0000\u04ee\u04ef\u0001\u0000\u0000\u0000"+ - "\u04ef\u04f0\u0006\u0097\u001f\u0000\u04f0\u013e\u0001\u0000\u0000\u0000"+ - "\u04f1\u04f2\u00039\u0015\u0000\u04f2\u04f3\u0001\u0000\u0000\u0000\u04f3"+ - "\u04f4\u0006\u0098\n\u0000\u04f4\u0140\u0001\u0000\u0000\u0000\u04f5\u04f6"+ - "\u0003;\u0016\u0000\u04f6\u04f7\u0001\u0000\u0000\u0000\u04f7\u04f8\u0006"+ - "\u0099\n\u0000\u04f8\u0142\u0001\u0000\u0000\u0000\u04f9\u04fa\u0003="+ - "\u0017\u0000\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u009a"+ - "\n\u0000\u04fc\u0144\u0001\u0000\u0000\u0000\u04fd\u04fe\u0003?\u0018"+ - "\u0000\u04fe\u04ff\u0001\u0000\u0000\u0000\u04ff\u0500\u0006\u009b\u000f"+ - "\u0000\u0500\u0501\u0006\u009b\u000b\u0000\u0501\u0146\u0001\u0000\u0000"+ - "\u0000\u0502\u0503\u0007\u0001\u0000\u0000\u0503\u0504\u0007\t\u0000\u0000"+ - "\u0504\u0505\u0007\u000f\u0000\u0000\u0505\u0506\u0007\u0007\u0000\u0000"+ - "\u0506\u0148\u0001\u0000\u0000\u0000\u0507\u0508\u00039\u0015\u0000\u0508"+ - "\u0509\u0001\u0000\u0000\u0000\u0509\u050a\u0006\u009d\n\u0000\u050a\u014a"+ - "\u0001\u0000\u0000\u0000\u050b\u050c\u0003;\u0016\u0000\u050c\u050d\u0001"+ - "\u0000\u0000\u0000\u050d\u050e\u0006\u009e\n\u0000\u050e\u014c\u0001\u0000"+ - "\u0000\u0000\u050f\u0510\u0003=\u0017\u0000\u0510\u0511\u0001\u0000\u0000"+ - "\u0000\u0511\u0512\u0006\u009f\n\u0000\u0512\u014e\u0001\u0000\u0000\u0000"+ - "\u0513\u0514\u0003\u00a7L\u0000\u0514\u0515\u0001\u0000\u0000\u0000\u0515"+ - "\u0516\u0006\u00a0\u0010\u0000\u0516\u0517\u0006\u00a0\u000b\u0000\u0517"+ - "\u0150\u0001\u0000\u0000\u0000\u0518\u0519\u0005:\u0000\u0000\u0519\u0152"+ - "\u0001\u0000\u0000\u0000\u051a\u0520\u0003K\u001e\u0000\u051b\u0520\u0003"+ - "A\u0019\u0000\u051c\u0520\u0003i-\u0000\u051d\u0520\u0003C\u001a\u0000"+ - "\u051e\u0520\u0003Q!\u0000\u051f\u051a\u0001\u0000\u0000\u0000\u051f\u051b"+ - "\u0001\u0000\u0000\u0000\u051f\u051c\u0001\u0000\u0000\u0000\u051f\u051d"+ - "\u0001\u0000\u0000\u0000\u051f\u051e\u0001\u0000\u0000\u0000\u0520\u0521"+ - "\u0001\u0000\u0000\u0000\u0521\u051f\u0001\u0000\u0000\u0000\u0521\u0522"+ - "\u0001\u0000\u0000\u0000\u0522\u0154\u0001\u0000\u0000\u0000\u0523\u0524"+ - "\u00039\u0015\u0000\u0524\u0525\u0001\u0000\u0000\u0000\u0525\u0526\u0006"+ - "\u00a3\n\u0000\u0526\u0156\u0001\u0000\u0000\u0000\u0527\u0528\u0003;"+ - "\u0016\u0000\u0528\u0529\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u00a4"+ - "\n\u0000\u052a\u0158\u0001\u0000\u0000\u0000\u052b\u052c\u0003=\u0017"+ - "\u0000\u052c\u052d\u0001\u0000\u0000\u0000\u052d\u052e\u0006\u00a5\n\u0000"+ - "\u052e\u015a\u0001\u0000\u0000\u0000\u052f\u0530\u0003?\u0018\u0000\u0530"+ - "\u0531\u0001\u0000\u0000\u0000\u0531\u0532\u0006\u00a6\u000f\u0000\u0532"+ - "\u0533\u0006\u00a6\u000b\u0000\u0533\u015c\u0001\u0000\u0000\u0000\u0534"+ - "\u0535\u0003\u0151\u00a1\u0000\u0535\u0536\u0001\u0000\u0000\u0000\u0536"+ - "\u0537\u0006\u00a7\u0011\u0000\u0537\u015e\u0001\u0000\u0000\u0000\u0538"+ - "\u0539\u0003e+\u0000\u0539\u053a\u0001\u0000\u0000\u0000\u053a\u053b\u0006"+ - "\u00a8\u0012\u0000\u053b\u0160\u0001\u0000\u0000\u0000\u053c\u053d\u0003"+ - "i-\u0000\u053d\u053e\u0001\u0000\u0000\u0000\u053e\u053f\u0006\u00a9\u0016"+ - "\u0000\u053f\u0162\u0001\u0000\u0000\u0000\u0540\u0541\u0003\u010b~\u0000"+ - "\u0541\u0542\u0001\u0000\u0000\u0000\u0542\u0543\u0006\u00aa \u0000\u0543"+ - "\u0544\u0006\u00aa!\u0000\u0544\u0164\u0001\u0000\u0000\u0000\u0545\u0546"+ - "\u0003\u00cf`\u0000\u0546\u0547\u0001\u0000\u0000\u0000\u0547\u0548\u0006"+ - "\u00ab\u0014\u0000\u0548\u0166\u0001\u0000\u0000\u0000\u0549\u054a\u0003"+ - "U#\u0000\u054a\u054b\u0001\u0000\u0000\u0000\u054b\u054c\u0006\u00ac\u0015"+ - "\u0000\u054c\u0168\u0001\u0000\u0000\u0000\u054d\u054e\u00039\u0015\u0000"+ - "\u054e\u054f\u0001\u0000\u0000\u0000\u054f\u0550\u0006\u00ad\n\u0000\u0550"+ - "\u016a\u0001\u0000\u0000\u0000\u0551\u0552\u0003;\u0016\u0000\u0552\u0553"+ - "\u0001\u0000\u0000\u0000\u0553\u0554\u0006\u00ae\n\u0000\u0554\u016c\u0001"+ - "\u0000\u0000\u0000\u0555\u0556\u0003=\u0017\u0000\u0556\u0557\u0001\u0000"+ - "\u0000\u0000\u0557\u0558\u0006\u00af\n\u0000\u0558\u016e\u0001\u0000\u0000"+ - "\u0000\u0559\u055a\u0003?\u0018\u0000\u055a\u055b\u0001\u0000\u0000\u0000"+ - "\u055b\u055c\u0006\u00b0\u000f\u0000\u055c\u055d\u0006\u00b0\u000b\u0000"+ - "\u055d\u055e\u0006\u00b0\u000b\u0000\u055e\u0170\u0001\u0000\u0000\u0000"+ - "\u055f\u0560\u0003e+\u0000\u0560\u0561\u0001\u0000\u0000\u0000\u0561\u0562"+ - "\u0006\u00b1\u0012\u0000\u0562\u0172\u0001\u0000\u0000\u0000\u0563\u0564"+ - "\u0003i-\u0000\u0564\u0565\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00b2"+ - "\u0016\u0000\u0566\u0174\u0001\u0000\u0000\u0000\u0567\u0568\u0003\u00e9"+ - "m\u0000\u0568\u0569\u0001\u0000\u0000\u0000\u0569\u056a\u0006\u00b3\u0019"+ - "\u0000\u056a\u0176\u0001\u0000\u0000\u0000\u056b\u056c\u00039\u0015\u0000"+ - "\u056c\u056d\u0001\u0000\u0000\u0000\u056d\u056e\u0006\u00b4\n\u0000\u056e"+ - "\u0178\u0001\u0000\u0000\u0000\u056f\u0570\u0003;\u0016\u0000\u0570\u0571"+ - "\u0001\u0000\u0000\u0000\u0571\u0572\u0006\u00b5\n\u0000\u0572\u017a\u0001"+ - "\u0000\u0000\u0000\u0573\u0574\u0003=\u0017\u0000\u0574\u0575\u0001\u0000"+ - "\u0000\u0000\u0575\u0576\u0006\u00b6\n\u0000\u0576\u017c\u0001\u0000\u0000"+ - "\u0000\u0577\u0578\u0003?\u0018\u0000\u0578\u0579\u0001\u0000\u0000\u0000"+ - "\u0579\u057a\u0006\u00b7\u000f\u0000\u057a\u057b\u0006\u00b7\u000b\u0000"+ - "\u057b\u017e\u0001\u0000\u0000\u0000\u057c\u057d\u0003\u00cf`\u0000\u057d"+ - "\u057e\u0001\u0000\u0000\u0000\u057e\u057f\u0006\u00b8\u0014\u0000\u057f"+ - "\u0580\u0006\u00b8\u000b\u0000\u0580\u0581\u0006\u00b8\"\u0000\u0581\u0180"+ - "\u0001\u0000\u0000\u0000\u0582\u0583\u0003U#\u0000\u0583\u0584\u0001\u0000"+ - "\u0000\u0000\u0584\u0585\u0006\u00b9\u0015\u0000\u0585\u0586\u0006\u00b9"+ - "\u000b\u0000\u0586\u0587\u0006\u00b9\"\u0000\u0587\u0182\u0001\u0000\u0000"+ - "\u0000\u0588\u0589\u00039\u0015\u0000\u0589\u058a\u0001\u0000\u0000\u0000"+ - "\u058a\u058b\u0006\u00ba\n\u0000\u058b\u0184\u0001\u0000\u0000\u0000\u058c"+ - "\u058d\u0003;\u0016\u0000\u058d\u058e\u0001\u0000\u0000\u0000\u058e\u058f"+ - "\u0006\u00bb\n\u0000\u058f\u0186\u0001\u0000\u0000\u0000\u0590\u0591\u0003"+ - "=\u0017\u0000\u0591\u0592\u0001\u0000\u0000\u0000\u0592\u0593\u0006\u00bc"+ - "\n\u0000\u0593\u0188\u0001\u0000\u0000\u0000\u0594\u0595\u0003\u0151\u00a1"+ - "\u0000\u0595\u0596\u0001\u0000\u0000\u0000\u0596\u0597\u0006\u00bd\u0011"+ - "\u0000\u0597\u0598\u0006\u00bd\u000b\u0000\u0598\u0599\u0006\u00bd\t\u0000"+ - "\u0599\u018a\u0001\u0000\u0000\u0000\u059a\u059b\u0003e+\u0000\u059b\u059c"+ - "\u0001\u0000\u0000\u0000\u059c\u059d\u0006\u00be\u0012\u0000\u059d\u059e"+ - "\u0006\u00be\u000b\u0000\u059e\u059f\u0006\u00be\t\u0000\u059f\u018c\u0001"+ - "\u0000\u0000\u0000\u05a0\u05a1\u00039\u0015\u0000\u05a1\u05a2\u0001\u0000"+ - "\u0000\u0000\u05a2\u05a3\u0006\u00bf\n\u0000\u05a3\u018e\u0001\u0000\u0000"+ - "\u0000\u05a4\u05a5\u0003;\u0016\u0000\u05a5\u05a6\u0001\u0000\u0000\u0000"+ - "\u05a6\u05a7\u0006\u00c0\n\u0000\u05a7\u0190\u0001\u0000\u0000\u0000\u05a8"+ - "\u05a9\u0003=\u0017\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa\u05ab"+ - "\u0006\u00c1\n\u0000\u05ab\u0192\u0001\u0000\u0000\u0000\u05ac\u05ad\u0003"+ - "\u00adO\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006\u00c2"+ - "\u000b\u0000\u05af\u05b0\u0006\u00c2\u0000\u0000\u05b0\u05b1\u0006\u00c2"+ - "\u001e\u0000\u05b1\u0194\u0001\u0000\u0000\u0000\u05b2\u05b3\u0003\u00a9"+ - "M\u0000\u05b3\u05b4\u0001\u0000\u0000\u0000\u05b4\u05b5\u0006\u00c3\u000b"+ - "\u0000\u05b5\u05b6\u0006\u00c3\u0000\u0000\u05b6\u05b7\u0006\u00c3\u001f"+ - "\u0000\u05b7\u0196\u0001\u0000\u0000\u0000\u05b8\u05b9\u0003[&\u0000\u05b9"+ - "\u05ba\u0001\u0000\u0000\u0000\u05ba\u05bb\u0006\u00c4\u000b\u0000\u05bb"+ - "\u05bc\u0006\u00c4\u0000\u0000\u05bc\u05bd\u0006\u00c4#\u0000\u05bd\u0198"+ - "\u0001\u0000\u0000\u0000\u05be\u05bf\u0003?\u0018\u0000\u05bf\u05c0\u0001"+ - "\u0000\u0000\u0000\u05c0\u05c1\u0006\u00c5\u000f\u0000\u05c1\u05c2\u0006"+ - "\u00c5\u000b\u0000\u05c2\u019a\u0001\u0000\u0000\u0000A\u0000\u0001\u0002"+ - "\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u024c\u0256\u025a"+ - "\u025d\u0266\u0268\u0273\u0286\u028b\u0294\u029b\u02a0\u02a2\u02ad\u02b5"+ - "\u02b8\u02ba\u02bf\u02c4\u02ca\u02d1\u02d6\u02dc\u02df\u02e7\u02eb\u036a"+ - "\u036f\u0376\u0378\u0388\u038d\u0392\u0394\u039a\u03e7\u03ec\u041b\u041f"+ - "\u0424\u0429\u042e\u0430\u0434\u0436\u048b\u048f\u0494\u051f\u0521$\u0005"+ - "\u0001\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005"+ - "\u0003\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b"+ - "\u0000\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0013\u0000"+ - "\u0007A\u0000\u0005\u0000\u0000\u0007\u0019\u0000\u0007B\u0000\u0007h"+ - "\u0000\u0007\"\u0000\u0007 \u0000\u0007L\u0000\u0007\u001a\u0000\u0007"+ - "$\u0000\u00070\u0000\u0007@\u0000\u0007P\u0000\u0005\n\u0000\u0005\u0007"+ + "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0013\u0004\u0013\u0242\b\u0013\u000b\u0013\f\u0013\u0243\u0001"+ + "\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0005"+ + "\u0014\u024c\b\u0014\n\u0014\f\u0014\u024f\t\u0014\u0001\u0014\u0003\u0014"+ + "\u0252\b\u0014\u0001\u0014\u0003\u0014\u0255\b\u0014\u0001\u0014\u0001"+ + "\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005"+ + "\u0015\u025e\b\u0015\n\u0015\f\u0015\u0261\t\u0015\u0001\u0015\u0001\u0015"+ + "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0004\u0016\u0269\b\u0016"+ + "\u000b\u0016\f\u0016\u026a\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017"+ + "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019"+ + "\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001c"+ + "\u0001\u001c\u0003\u001c\u027e\b\u001c\u0001\u001c\u0004\u001c\u0281\b"+ + "\u001c\u000b\u001c\f\u001c\u0282\u0001\u001d\u0001\u001d\u0001\u001e\u0001"+ + "\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u028c\b\u001f\u0001"+ + " \u0001 \u0001!\u0001!\u0001!\u0003!\u0293\b!\u0001\"\u0001\"\u0001\""+ + "\u0005\"\u0298\b\"\n\"\f\"\u029b\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ + "\"\u0001\"\u0005\"\u02a3\b\"\n\"\f\"\u02a6\t\"\u0001\"\u0001\"\u0001\""+ + "\u0001\"\u0001\"\u0003\"\u02ad\b\"\u0001\"\u0003\"\u02b0\b\"\u0003\"\u02b2"+ + "\b\"\u0001#\u0004#\u02b5\b#\u000b#\f#\u02b6\u0001$\u0004$\u02ba\b$\u000b"+ + "$\f$\u02bb\u0001$\u0001$\u0005$\u02c0\b$\n$\f$\u02c3\t$\u0001$\u0001$"+ + "\u0004$\u02c7\b$\u000b$\f$\u02c8\u0001$\u0004$\u02cc\b$\u000b$\f$\u02cd"+ + "\u0001$\u0001$\u0005$\u02d2\b$\n$\f$\u02d5\t$\u0003$\u02d7\b$\u0001$\u0001"+ + "$\u0001$\u0001$\u0004$\u02dd\b$\u000b$\f$\u02de\u0001$\u0001$\u0003$\u02e3"+ + "\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001"+ + "\'\u0001\'\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001+\u0001"+ + "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001"+ + "-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001"+ + "/\u00010\u00010\u00010\u00011\u00011\u00011\u00011\u00011\u00012\u0001"+ + "2\u00012\u00012\u00012\u00013\u00013\u00014\u00014\u00014\u00014\u0001"+ + "5\u00015\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00016\u0001"+ + "6\u00017\u00017\u00017\u00018\u00018\u00019\u00019\u00019\u00019\u0001"+ + "9\u00019\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001"+ + "<\u0001<\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001"+ + "@\u0001@\u0001@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001"+ + "D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001"+ + "H\u0001H\u0001H\u0001H\u0001I\u0001I\u0001I\u0001I\u0001I\u0001J\u0001"+ + "J\u0001J\u0003J\u0368\bJ\u0001J\u0005J\u036b\bJ\nJ\fJ\u036e\tJ\u0001J"+ + "\u0001J\u0004J\u0372\bJ\u000bJ\fJ\u0373\u0003J\u0376\bJ\u0001K\u0001K"+ + "\u0001K\u0001K\u0001K\u0001L\u0001L\u0001L\u0001L\u0001L\u0001M\u0001"+ + "M\u0005M\u0384\bM\nM\fM\u0387\tM\u0001M\u0001M\u0003M\u038b\bM\u0001M"+ + "\u0004M\u038e\bM\u000bM\fM\u038f\u0003M\u0392\bM\u0001N\u0001N\u0004N"+ + "\u0396\bN\u000bN\fN\u0397\u0001N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001"+ + "P\u0001P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001"+ + "S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001T\u0001T\u0001"+ + "U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001"+ + "W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001"+ + "Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001"+ + "\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001"+ + "^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0003_\u03e5"+ + "\b_\u0001`\u0004`\u03e8\b`\u000b`\f`\u03e9\u0001a\u0001a\u0001a\u0001"+ + "a\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001"+ + "d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001"+ + "f\u0001f\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001"+ + "i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001"+ + "k\u0001k\u0003k\u0419\bk\u0001l\u0001l\u0003l\u041d\bl\u0001l\u0005l\u0420"+ + "\bl\nl\fl\u0423\tl\u0001l\u0001l\u0003l\u0427\bl\u0001l\u0004l\u042a\b"+ + "l\u000bl\fl\u042b\u0003l\u042e\bl\u0001m\u0001m\u0004m\u0432\bm\u000b"+ + "m\fm\u0433\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001"+ + "p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001q\u0001r\u0001"+ + "r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001"+ + "t\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001"+ + "w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001"+ + "z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001"+ + "|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001"+ + "~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001"+ + "\u007f\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0081\u0004"+ + "\u0081\u0487\b\u0081\u000b\u0081\f\u0081\u0488\u0001\u0081\u0001\u0081"+ + "\u0003\u0081\u048d\b\u0081\u0001\u0081\u0004\u0081\u0490\b\u0081\u000b"+ + "\u0081\f\u0081\u0491\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001"+ + "\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001"+ + "\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001"+ + "\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001"+ + "\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001"+ + "\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001"+ + "\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001"+ + "\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+ + "\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e\u0001"+ + "\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001"+ + "\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001\u0091\u0001"+ + "\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001"+ + "\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0094\u0001"+ + "\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001\u0095\u0001"+ + "\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001"+ + "\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098\u0001"+ + "\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u009a\u0001"+ + "\u009a\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001\u009b\u0001"+ + "\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001"+ + "\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e\u0001"+ + "\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001"+ + "\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001"+ + "\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001"+ + "\u00a2\u0004\u00a2\u051d\b\u00a2\u000b\u00a2\f\u00a2\u051e\u0001\u00a3"+ + "\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4"+ + "\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6"+ + "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7"+ + "\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8"+ + "\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001\u00aa"+ + "\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001\u00ab"+ + "\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ad"+ + "\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae\u0001\u00ae"+ + "\u0001\u00ae\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00b0"+ + "\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b1"+ + "\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2\u0001\u00b2"+ + "\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b4"+ + "\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5\u0001\u00b5"+ + "\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b7"+ + "\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8\u0001\u00b8"+ + "\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001\u00b9"+ + "\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba"+ + "\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb"+ + "\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd"+ + "\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be"+ + "\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf\u0001\u00bf"+ + "\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0"+ + "\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2"+ + "\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3\u0001\u00c3"+ + "\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4\u0001\u00c4"+ + "\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5\u0001\u00c5"+ + "\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0002\u025f\u02a4\u0000\u00c6\u000f"+ + "\u0001\u0011\u0002\u0013\u0003\u0015\u0004\u0017\u0005\u0019\u0006\u001b"+ + "\u0007\u001d\b\u001f\t!\n#\u000b%\f\'\r)\u000e+\u000f-\u0010/\u00111\u0012"+ + "3\u00135\u00147\u00159\u0016;\u0017=\u0018?\u0000A\u0000C\u0000E\u0000"+ + "G\u0000I\u0000K\u0000M\u0000O\u0000Q\u0000S\u0019U\u001aW\u001bY\u001c"+ + "[\u001d]\u001e_\u001fa c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f/\u00810\u0083"+ + "1\u00852\u00873\u00894\u008b5\u008d6\u008f7\u00918\u00939\u0095:\u0097"+ + ";\u0099<\u009b=\u009d>\u009f?\u00a1\u0000\u00a3@\u00a5A\u00a7B\u00a9C"+ + "\u00ab\u0000\u00adD\u00afE\u00b1F\u00b3G\u00b5\u0000\u00b7\u0000\u00b9"+ + "H\u00bbI\u00bdJ\u00bf\u0000\u00c1\u0000\u00c3\u0000\u00c5\u0000\u00c7"+ + "\u0000\u00c9\u0000\u00cbK\u00cd\u0000\u00cfL\u00d1\u0000\u00d3\u0000\u00d5"+ + "M\u00d7N\u00d9O\u00db\u0000\u00dd\u0000\u00df\u0000\u00e1\u0000\u00e3"+ + "\u0000\u00e5\u0000\u00e7\u0000\u00e9P\u00ebQ\u00edR\u00efS\u00f1\u0000"+ + "\u00f3\u0000\u00f5\u0000\u00f7\u0000\u00f9\u0000\u00fb\u0000\u00fdT\u00ff"+ + "\u0000\u0101U\u0103V\u0105W\u0107\u0000\u0109\u0000\u010bX\u010dY\u010f"+ + "\u0000\u0111Z\u0113\u0000\u0115[\u0117\\\u0119]\u011b\u0000\u011d\u0000"+ + "\u011f\u0000\u0121\u0000\u0123\u0000\u0125\u0000\u0127\u0000\u0129\u0000"+ + "\u012b\u0000\u012d^\u012f_\u0131`\u0133\u0000\u0135\u0000\u0137\u0000"+ + "\u0139\u0000\u013b\u0000\u013d\u0000\u013fa\u0141b\u0143c\u0145\u0000"+ + "\u0147d\u0149e\u014bf\u014dg\u014f\u0000\u0151h\u0153i\u0155j\u0157k\u0159"+ + "l\u015b\u0000\u015d\u0000\u015f\u0000\u0161\u0000\u0163\u0000\u0165\u0000"+ + "\u0167\u0000\u0169m\u016bn\u016do\u016f\u0000\u0171\u0000\u0173\u0000"+ + "\u0175\u0000\u0177p\u0179q\u017br\u017d\u0000\u017f\u0000\u0181\u0000"+ + "\u0183s\u0185t\u0187u\u0189\u0000\u018b\u0000\u018dv\u018fw\u0191x\u0193"+ + "\u0000\u0195\u0000\u0197\u0000\u0199\u0000\u000f\u0000\u0001\u0002\u0003"+ + "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e#\u0002\u0000DDdd\u0002"+ + "\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002\u0000"+ + "TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000PPpp\u0002\u0000NNnn\u0002"+ + "\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002\u0000"+ + "XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000GGgg\u0002\u0000KKkk\u0002"+ + "\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r\r //[[]]\u0002\u0000\n\n"+ + "\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002\u0000AZaz\b\u0000\"\"N"+ + "NRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001\u0000"+ + "``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t\n\r\r \"\",,//::==["+ + "[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r \"#,,//::<<>?\\\\||\u05dc\u0000"+ + "\u000f\u0001\u0000\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000"+ + "\u0013\u0001\u0000\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000"+ + "\u0017\u0001\u0000\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000"+ + "\u001b\u0001\u0000\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000"+ + "\u001f\u0001\u0000\u0000\u0000\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001"+ + "\u0000\u0000\u0000\u0000%\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000"+ + "\u0000\u0000\u0000)\u0001\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000"+ + "\u0000-\u0001\u0000\u0000\u0000\u0000/\u0001\u0000\u0000\u0000\u00001"+ + "\u0001\u0000\u0000\u0000\u00003\u0001\u0000\u0000\u0000\u00005\u0001\u0000"+ + "\u0000\u0000\u00007\u0001\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000"+ + "\u0000;\u0001\u0000\u0000\u0000\u0001=\u0001\u0000\u0000\u0000\u0001S"+ + "\u0001\u0000\u0000\u0000\u0001U\u0001\u0000\u0000\u0000\u0001W\u0001\u0000"+ + "\u0000\u0000\u0001Y\u0001\u0000\u0000\u0000\u0001[\u0001\u0000\u0000\u0000"+ + "\u0001]\u0001\u0000\u0000\u0000\u0001_\u0001\u0000\u0000\u0000\u0001a"+ + "\u0001\u0000\u0000\u0000\u0001c\u0001\u0000\u0000\u0000\u0001e\u0001\u0000"+ + "\u0000\u0000\u0001g\u0001\u0000\u0000\u0000\u0001i\u0001\u0000\u0000\u0000"+ + "\u0001k\u0001\u0000\u0000\u0000\u0001m\u0001\u0000\u0000\u0000\u0001o"+ + "\u0001\u0000\u0000\u0000\u0001q\u0001\u0000\u0000\u0000\u0001s\u0001\u0000"+ + "\u0000\u0000\u0001u\u0001\u0000\u0000\u0000\u0001w\u0001\u0000\u0000\u0000"+ + "\u0001y\u0001\u0000\u0000\u0000\u0001{\u0001\u0000\u0000\u0000\u0001}"+ + "\u0001\u0000\u0000\u0000\u0001\u007f\u0001\u0000\u0000\u0000\u0001\u0081"+ + "\u0001\u0000\u0000\u0000\u0001\u0083\u0001\u0000\u0000\u0000\u0001\u0085"+ + "\u0001\u0000\u0000\u0000\u0001\u0087\u0001\u0000\u0000\u0000\u0001\u0089"+ + "\u0001\u0000\u0000\u0000\u0001\u008b\u0001\u0000\u0000\u0000\u0001\u008d"+ + "\u0001\u0000\u0000\u0000\u0001\u008f\u0001\u0000\u0000\u0000\u0001\u0091"+ + "\u0001\u0000\u0000\u0000\u0001\u0093\u0001\u0000\u0000\u0000\u0001\u0095"+ + "\u0001\u0000\u0000\u0000\u0001\u0097\u0001\u0000\u0000\u0000\u0001\u0099"+ + "\u0001\u0000\u0000\u0000\u0001\u009b\u0001\u0000\u0000\u0000\u0001\u009d"+ + "\u0001\u0000\u0000\u0000\u0001\u009f\u0001\u0000\u0000\u0000\u0001\u00a1"+ + "\u0001\u0000\u0000\u0000\u0001\u00a3\u0001\u0000\u0000\u0000\u0001\u00a5"+ + "\u0001\u0000\u0000\u0000\u0001\u00a7\u0001\u0000\u0000\u0000\u0001\u00a9"+ + "\u0001\u0000\u0000\u0000\u0001\u00ad\u0001\u0000\u0000\u0000\u0001\u00af"+ + "\u0001\u0000\u0000\u0000\u0001\u00b1\u0001\u0000\u0000\u0000\u0001\u00b3"+ + "\u0001\u0000\u0000\u0000\u0002\u00b5\u0001\u0000\u0000\u0000\u0002\u00b7"+ + "\u0001\u0000\u0000\u0000\u0002\u00b9\u0001\u0000\u0000\u0000\u0002\u00bb"+ + "\u0001\u0000\u0000\u0000\u0002\u00bd\u0001\u0000\u0000\u0000\u0003\u00bf"+ + "\u0001\u0000\u0000\u0000\u0003\u00c1\u0001\u0000\u0000\u0000\u0003\u00c3"+ + "\u0001\u0000\u0000\u0000\u0003\u00c5\u0001\u0000\u0000\u0000\u0003\u00c7"+ + "\u0001\u0000\u0000\u0000\u0003\u00c9\u0001\u0000\u0000\u0000\u0003\u00cb"+ + "\u0001\u0000\u0000\u0000\u0003\u00cf\u0001\u0000\u0000\u0000\u0003\u00d1"+ + "\u0001\u0000\u0000\u0000\u0003\u00d3\u0001\u0000\u0000\u0000\u0003\u00d5"+ + "\u0001\u0000\u0000\u0000\u0003\u00d7\u0001\u0000\u0000\u0000\u0003\u00d9"+ + "\u0001\u0000\u0000\u0000\u0004\u00db\u0001\u0000\u0000\u0000\u0004\u00dd"+ + "\u0001\u0000\u0000\u0000\u0004\u00df\u0001\u0000\u0000\u0000\u0004\u00e1"+ + "\u0001\u0000\u0000\u0000\u0004\u00e3\u0001\u0000\u0000\u0000\u0004\u00e9"+ + "\u0001\u0000\u0000\u0000\u0004\u00eb\u0001\u0000\u0000\u0000\u0004\u00ed"+ + "\u0001\u0000\u0000\u0000\u0004\u00ef\u0001\u0000\u0000\u0000\u0005\u00f1"+ + "\u0001\u0000\u0000\u0000\u0005\u00f3\u0001\u0000\u0000\u0000\u0005\u00f5"+ + "\u0001\u0000\u0000\u0000\u0005\u00f7\u0001\u0000\u0000\u0000\u0005\u00f9"+ + "\u0001\u0000\u0000\u0000\u0005\u00fb\u0001\u0000\u0000\u0000\u0005\u00fd"+ + "\u0001\u0000\u0000\u0000\u0005\u00ff\u0001\u0000\u0000\u0000\u0005\u0101"+ + "\u0001\u0000\u0000\u0000\u0005\u0103\u0001\u0000\u0000\u0000\u0005\u0105"+ + "\u0001\u0000\u0000\u0000\u0006\u0107\u0001\u0000\u0000\u0000\u0006\u0109"+ + "\u0001\u0000\u0000\u0000\u0006\u010b\u0001\u0000\u0000\u0000\u0006\u010d"+ + "\u0001\u0000\u0000\u0000\u0006\u0111\u0001\u0000\u0000\u0000\u0006\u0113"+ + "\u0001\u0000\u0000\u0000\u0006\u0115\u0001\u0000\u0000\u0000\u0006\u0117"+ + "\u0001\u0000\u0000\u0000\u0006\u0119\u0001\u0000\u0000\u0000\u0007\u011b"+ + "\u0001\u0000\u0000\u0000\u0007\u011d\u0001\u0000\u0000\u0000\u0007\u011f"+ + "\u0001\u0000\u0000\u0000\u0007\u0121\u0001\u0000\u0000\u0000\u0007\u0123"+ + "\u0001\u0000\u0000\u0000\u0007\u0125\u0001\u0000\u0000\u0000\u0007\u0127"+ + "\u0001\u0000\u0000\u0000\u0007\u0129\u0001\u0000\u0000\u0000\u0007\u012b"+ + "\u0001\u0000\u0000\u0000\u0007\u012d\u0001\u0000\u0000\u0000\u0007\u012f"+ + "\u0001\u0000\u0000\u0000\u0007\u0131\u0001\u0000\u0000\u0000\b\u0133\u0001"+ + "\u0000\u0000\u0000\b\u0135\u0001\u0000\u0000\u0000\b\u0137\u0001\u0000"+ + "\u0000\u0000\b\u0139\u0001\u0000\u0000\u0000\b\u013b\u0001\u0000\u0000"+ + "\u0000\b\u013d\u0001\u0000\u0000\u0000\b\u013f\u0001\u0000\u0000\u0000"+ + "\b\u0141\u0001\u0000\u0000\u0000\b\u0143\u0001\u0000\u0000\u0000\t\u0145"+ + "\u0001\u0000\u0000\u0000\t\u0147\u0001\u0000\u0000\u0000\t\u0149\u0001"+ + "\u0000\u0000\u0000\t\u014b\u0001\u0000\u0000\u0000\t\u014d\u0001\u0000"+ + "\u0000\u0000\n\u014f\u0001\u0000\u0000\u0000\n\u0151\u0001\u0000\u0000"+ + "\u0000\n\u0153\u0001\u0000\u0000\u0000\n\u0155\u0001\u0000\u0000\u0000"+ + "\n\u0157\u0001\u0000\u0000\u0000\n\u0159\u0001\u0000\u0000\u0000\u000b"+ + "\u015b\u0001\u0000\u0000\u0000\u000b\u015d\u0001\u0000\u0000\u0000\u000b"+ + "\u015f\u0001\u0000\u0000\u0000\u000b\u0161\u0001\u0000\u0000\u0000\u000b"+ + "\u0163\u0001\u0000\u0000\u0000\u000b\u0165\u0001\u0000\u0000\u0000\u000b"+ + "\u0167\u0001\u0000\u0000\u0000\u000b\u0169\u0001\u0000\u0000\u0000\u000b"+ + "\u016b\u0001\u0000\u0000\u0000\u000b\u016d\u0001\u0000\u0000\u0000\f\u016f"+ + "\u0001\u0000\u0000\u0000\f\u0171\u0001\u0000\u0000\u0000\f\u0173\u0001"+ + "\u0000\u0000\u0000\f\u0175\u0001\u0000\u0000\u0000\f\u0177\u0001\u0000"+ + "\u0000\u0000\f\u0179\u0001\u0000\u0000\u0000\f\u017b\u0001\u0000\u0000"+ + "\u0000\r\u017d\u0001\u0000\u0000\u0000\r\u017f\u0001\u0000\u0000\u0000"+ + "\r\u0181\u0001\u0000\u0000\u0000\r\u0183\u0001\u0000\u0000\u0000\r\u0185"+ + "\u0001\u0000\u0000\u0000\r\u0187\u0001\u0000\u0000\u0000\u000e\u0189\u0001"+ + "\u0000\u0000\u0000\u000e\u018b\u0001\u0000\u0000\u0000\u000e\u018d\u0001"+ + "\u0000\u0000\u0000\u000e\u018f\u0001\u0000\u0000\u0000\u000e\u0191\u0001"+ + "\u0000\u0000\u0000\u000e\u0193\u0001\u0000\u0000\u0000\u000e\u0195\u0001"+ + "\u0000\u0000\u0000\u000e\u0197\u0001\u0000\u0000\u0000\u000e\u0199\u0001"+ + "\u0000\u0000\u0000\u000f\u019b\u0001\u0000\u0000\u0000\u0011\u01a5\u0001"+ + "\u0000\u0000\u0000\u0013\u01ac\u0001\u0000\u0000\u0000\u0015\u01b5\u0001"+ + "\u0000\u0000\u0000\u0017\u01bc\u0001\u0000\u0000\u0000\u0019\u01c6\u0001"+ + "\u0000\u0000\u0000\u001b\u01cd\u0001\u0000\u0000\u0000\u001d\u01d4\u0001"+ + "\u0000\u0000\u0000\u001f\u01db\u0001\u0000\u0000\u0000!\u01e3\u0001\u0000"+ + "\u0000\u0000#\u01ef\u0001\u0000\u0000\u0000%\u01f8\u0001\u0000\u0000\u0000"+ + "\'\u01fe\u0001\u0000\u0000\u0000)\u0205\u0001\u0000\u0000\u0000+\u020c"+ + "\u0001\u0000\u0000\u0000-\u0214\u0001\u0000\u0000\u0000/\u021c\u0001\u0000"+ + "\u0000\u00001\u022b\u0001\u0000\u0000\u00003\u0235\u0001\u0000\u0000\u0000"+ + "5\u0241\u0001\u0000\u0000\u00007\u0247\u0001\u0000\u0000\u00009\u0258"+ + "\u0001\u0000\u0000\u0000;\u0268\u0001\u0000\u0000\u0000=\u026e\u0001\u0000"+ + "\u0000\u0000?\u0272\u0001\u0000\u0000\u0000A\u0274\u0001\u0000\u0000\u0000"+ + "C\u0276\u0001\u0000\u0000\u0000E\u0279\u0001\u0000\u0000\u0000G\u027b"+ + "\u0001\u0000\u0000\u0000I\u0284\u0001\u0000\u0000\u0000K\u0286\u0001\u0000"+ + "\u0000\u0000M\u028b\u0001\u0000\u0000\u0000O\u028d\u0001\u0000\u0000\u0000"+ + "Q\u0292\u0001\u0000\u0000\u0000S\u02b1\u0001\u0000\u0000\u0000U\u02b4"+ + "\u0001\u0000\u0000\u0000W\u02e2\u0001\u0000\u0000\u0000Y\u02e4\u0001\u0000"+ + "\u0000\u0000[\u02e7\u0001\u0000\u0000\u0000]\u02eb\u0001\u0000\u0000\u0000"+ + "_\u02ef\u0001\u0000\u0000\u0000a\u02f1\u0001\u0000\u0000\u0000c\u02f4"+ + "\u0001\u0000\u0000\u0000e\u02f6\u0001\u0000\u0000\u0000g\u02fb\u0001\u0000"+ + "\u0000\u0000i\u02fd\u0001\u0000\u0000\u0000k\u0303\u0001\u0000\u0000\u0000"+ + "m\u0309\u0001\u0000\u0000\u0000o\u030c\u0001\u0000\u0000\u0000q\u030f"+ + "\u0001\u0000\u0000\u0000s\u0314\u0001\u0000\u0000\u0000u\u0319\u0001\u0000"+ + "\u0000\u0000w\u031b\u0001\u0000\u0000\u0000y\u031f\u0001\u0000\u0000\u0000"+ + "{\u0324\u0001\u0000\u0000\u0000}\u032a\u0001\u0000\u0000\u0000\u007f\u032d"+ + "\u0001\u0000\u0000\u0000\u0081\u032f\u0001\u0000\u0000\u0000\u0083\u0335"+ + "\u0001\u0000\u0000\u0000\u0085\u0337\u0001\u0000\u0000\u0000\u0087\u033c"+ + "\u0001\u0000\u0000\u0000\u0089\u033f\u0001\u0000\u0000\u0000\u008b\u0342"+ + "\u0001\u0000\u0000\u0000\u008d\u0345\u0001\u0000\u0000\u0000\u008f\u0347"+ + "\u0001\u0000\u0000\u0000\u0091\u034a\u0001\u0000\u0000\u0000\u0093\u034c"+ + "\u0001\u0000\u0000\u0000\u0095\u034f\u0001\u0000\u0000\u0000\u0097\u0351"+ + "\u0001\u0000\u0000\u0000\u0099\u0353\u0001\u0000\u0000\u0000\u009b\u0355"+ + "\u0001\u0000\u0000\u0000\u009d\u0357\u0001\u0000\u0000\u0000\u009f\u0359"+ + "\u0001\u0000\u0000\u0000\u00a1\u035f\u0001\u0000\u0000\u0000\u00a3\u0375"+ + "\u0001\u0000\u0000\u0000\u00a5\u0377\u0001\u0000\u0000\u0000\u00a7\u037c"+ + "\u0001\u0000\u0000\u0000\u00a9\u0391\u0001\u0000\u0000\u0000\u00ab\u0393"+ + "\u0001\u0000\u0000\u0000\u00ad\u039b\u0001\u0000\u0000\u0000\u00af\u039d"+ + "\u0001\u0000\u0000\u0000\u00b1\u03a1\u0001\u0000\u0000\u0000\u00b3\u03a5"+ + "\u0001\u0000\u0000\u0000\u00b5\u03a9\u0001\u0000\u0000\u0000\u00b7\u03ae"+ + "\u0001\u0000\u0000\u0000\u00b9\u03b3\u0001\u0000\u0000\u0000\u00bb\u03b7"+ + "\u0001\u0000\u0000\u0000\u00bd\u03bb\u0001\u0000\u0000\u0000\u00bf\u03bf"+ + "\u0001\u0000\u0000\u0000\u00c1\u03c4\u0001\u0000\u0000\u0000\u00c3\u03c8"+ + "\u0001\u0000\u0000\u0000\u00c5\u03cc\u0001\u0000\u0000\u0000\u00c7\u03d0"+ + "\u0001\u0000\u0000\u0000\u00c9\u03d4\u0001\u0000\u0000\u0000\u00cb\u03d8"+ + "\u0001\u0000\u0000\u0000\u00cd\u03e4\u0001\u0000\u0000\u0000\u00cf\u03e7"+ + "\u0001\u0000\u0000\u0000\u00d1\u03eb\u0001\u0000\u0000\u0000\u00d3\u03ef"+ + "\u0001\u0000\u0000\u0000\u00d5\u03f3\u0001\u0000\u0000\u0000\u00d7\u03f7"+ + "\u0001\u0000\u0000\u0000\u00d9\u03fb\u0001\u0000\u0000\u0000\u00db\u03ff"+ + "\u0001\u0000\u0000\u0000\u00dd\u0404\u0001\u0000\u0000\u0000\u00df\u0408"+ + "\u0001\u0000\u0000\u0000\u00e1\u040c\u0001\u0000\u0000\u0000\u00e3\u0410"+ + "\u0001\u0000\u0000\u0000\u00e5\u0418\u0001\u0000\u0000\u0000\u00e7\u042d"+ + "\u0001\u0000\u0000\u0000\u00e9\u0431\u0001\u0000\u0000\u0000\u00eb\u0435"+ + "\u0001\u0000\u0000\u0000\u00ed\u0439\u0001\u0000\u0000\u0000\u00ef\u043d"+ + "\u0001\u0000\u0000\u0000\u00f1\u0441\u0001\u0000\u0000\u0000\u00f3\u0446"+ + "\u0001\u0000\u0000\u0000\u00f5\u044a\u0001\u0000\u0000\u0000\u00f7\u044e"+ + "\u0001\u0000\u0000\u0000\u00f9\u0452\u0001\u0000\u0000\u0000\u00fb\u0456"+ + "\u0001\u0000\u0000\u0000\u00fd\u045a\u0001\u0000\u0000\u0000\u00ff\u045d"+ + "\u0001\u0000\u0000\u0000\u0101\u0461\u0001\u0000\u0000\u0000\u0103\u0465"+ + "\u0001\u0000\u0000\u0000\u0105\u0469\u0001\u0000\u0000\u0000\u0107\u046d"+ + "\u0001\u0000\u0000\u0000\u0109\u0472\u0001\u0000\u0000\u0000\u010b\u0477"+ + "\u0001\u0000\u0000\u0000\u010d\u047c\u0001\u0000\u0000\u0000\u010f\u0483"+ + "\u0001\u0000\u0000\u0000\u0111\u048c\u0001\u0000\u0000\u0000\u0113\u0493"+ + "\u0001\u0000\u0000\u0000\u0115\u0497\u0001\u0000\u0000\u0000\u0117\u049b"+ + "\u0001\u0000\u0000\u0000\u0119\u049f\u0001\u0000\u0000\u0000\u011b\u04a3"+ + "\u0001\u0000\u0000\u0000\u011d\u04a9\u0001\u0000\u0000\u0000\u011f\u04ad"+ + "\u0001\u0000\u0000\u0000\u0121\u04b1\u0001\u0000\u0000\u0000\u0123\u04b5"+ + "\u0001\u0000\u0000\u0000\u0125\u04b9\u0001\u0000\u0000\u0000\u0127\u04bd"+ + "\u0001\u0000\u0000\u0000\u0129\u04c1\u0001\u0000\u0000\u0000\u012b\u04c5"+ + "\u0001\u0000\u0000\u0000\u012d\u04c9\u0001\u0000\u0000\u0000\u012f\u04cd"+ + "\u0001\u0000\u0000\u0000\u0131\u04d1\u0001\u0000\u0000\u0000\u0133\u04d5"+ + "\u0001\u0000\u0000\u0000\u0135\u04da\u0001\u0000\u0000\u0000\u0137\u04de"+ + "\u0001\u0000\u0000\u0000\u0139\u04e2\u0001\u0000\u0000\u0000\u013b\u04e6"+ + "\u0001\u0000\u0000\u0000\u013d\u04ea\u0001\u0000\u0000\u0000\u013f\u04ee"+ + "\u0001\u0000\u0000\u0000\u0141\u04f2\u0001\u0000\u0000\u0000\u0143\u04f6"+ + "\u0001\u0000\u0000\u0000\u0145\u04fa\u0001\u0000\u0000\u0000\u0147\u04ff"+ + "\u0001\u0000\u0000\u0000\u0149\u0504\u0001\u0000\u0000\u0000\u014b\u0508"+ + "\u0001\u0000\u0000\u0000\u014d\u050c\u0001\u0000\u0000\u0000\u014f\u0510"+ + "\u0001\u0000\u0000\u0000\u0151\u0515\u0001\u0000\u0000\u0000\u0153\u051c"+ + "\u0001\u0000\u0000\u0000\u0155\u0520\u0001\u0000\u0000\u0000\u0157\u0524"+ + "\u0001\u0000\u0000\u0000\u0159\u0528\u0001\u0000\u0000\u0000\u015b\u052c"+ + "\u0001\u0000\u0000\u0000\u015d\u0531\u0001\u0000\u0000\u0000\u015f\u0535"+ + "\u0001\u0000\u0000\u0000\u0161\u0539\u0001\u0000\u0000\u0000\u0163\u053d"+ + "\u0001\u0000\u0000\u0000\u0165\u0542\u0001\u0000\u0000\u0000\u0167\u0546"+ + "\u0001\u0000\u0000\u0000\u0169\u054a\u0001\u0000\u0000\u0000\u016b\u054e"+ + "\u0001\u0000\u0000\u0000\u016d\u0552\u0001\u0000\u0000\u0000\u016f\u0556"+ + "\u0001\u0000\u0000\u0000\u0171\u055c\u0001\u0000\u0000\u0000\u0173\u0560"+ + "\u0001\u0000\u0000\u0000\u0175\u0564\u0001\u0000\u0000\u0000\u0177\u0568"+ + "\u0001\u0000\u0000\u0000\u0179\u056c\u0001\u0000\u0000\u0000\u017b\u0570"+ + "\u0001\u0000\u0000\u0000\u017d\u0574\u0001\u0000\u0000\u0000\u017f\u0579"+ + "\u0001\u0000\u0000\u0000\u0181\u057f\u0001\u0000\u0000\u0000\u0183\u0585"+ + "\u0001\u0000\u0000\u0000\u0185\u0589\u0001\u0000\u0000\u0000\u0187\u058d"+ + "\u0001\u0000\u0000\u0000\u0189\u0591\u0001\u0000\u0000\u0000\u018b\u0597"+ + "\u0001\u0000\u0000\u0000\u018d\u059d\u0001\u0000\u0000\u0000\u018f\u05a1"+ + "\u0001\u0000\u0000\u0000\u0191\u05a5\u0001\u0000\u0000\u0000\u0193\u05a9"+ + "\u0001\u0000\u0000\u0000\u0195\u05af\u0001\u0000\u0000\u0000\u0197\u05b5"+ + "\u0001\u0000\u0000\u0000\u0199\u05bb\u0001\u0000\u0000\u0000\u019b\u019c"+ + "\u0007\u0000\u0000\u0000\u019c\u019d\u0007\u0001\u0000\u0000\u019d\u019e"+ + "\u0007\u0002\u0000\u0000\u019e\u019f\u0007\u0002\u0000\u0000\u019f\u01a0"+ + "\u0007\u0003\u0000\u0000\u01a0\u01a1\u0007\u0004\u0000\u0000\u01a1\u01a2"+ + "\u0007\u0005\u0000\u0000\u01a2\u01a3\u0001\u0000\u0000\u0000\u01a3\u01a4"+ + "\u0006\u0000\u0000\u0000\u01a4\u0010\u0001\u0000\u0000\u0000\u01a5\u01a6"+ + "\u0007\u0000\u0000\u0000\u01a6\u01a7\u0007\u0006\u0000\u0000\u01a7\u01a8"+ + "\u0007\u0007\u0000\u0000\u01a8\u01a9\u0007\b\u0000\u0000\u01a9\u01aa\u0001"+ + "\u0000\u0000\u0000\u01aa\u01ab\u0006\u0001\u0001\u0000\u01ab\u0012\u0001"+ + "\u0000\u0000\u0000\u01ac\u01ad\u0007\u0003\u0000\u0000\u01ad\u01ae\u0007"+ + "\t\u0000\u0000\u01ae\u01af\u0007\u0006\u0000\u0000\u01af\u01b0\u0007\u0001"+ + "\u0000\u0000\u01b0\u01b1\u0007\u0004\u0000\u0000\u01b1\u01b2\u0007\n\u0000"+ + "\u0000\u01b2\u01b3\u0001\u0000\u0000\u0000\u01b3\u01b4\u0006\u0002\u0002"+ + "\u0000\u01b4\u0014\u0001\u0000\u0000\u0000\u01b5\u01b6\u0007\u0003\u0000"+ + "\u0000\u01b6\u01b7\u0007\u000b\u0000\u0000\u01b7\u01b8\u0007\f\u0000\u0000"+ + "\u01b8\u01b9\u0007\r\u0000\u0000\u01b9\u01ba\u0001\u0000\u0000\u0000\u01ba"+ + "\u01bb\u0006\u0003\u0000\u0000\u01bb\u0016\u0001\u0000\u0000\u0000\u01bc"+ + "\u01bd\u0007\u0003\u0000\u0000\u01bd\u01be\u0007\u000e\u0000\u0000\u01be"+ + "\u01bf\u0007\b\u0000\u0000\u01bf\u01c0\u0007\r\u0000\u0000\u01c0\u01c1"+ + "\u0007\f\u0000\u0000\u01c1\u01c2\u0007\u0001\u0000\u0000\u01c2\u01c3\u0007"+ + "\t\u0000\u0000\u01c3\u01c4\u0001\u0000\u0000\u0000\u01c4\u01c5\u0006\u0004"+ + "\u0003\u0000\u01c5\u0018\u0001\u0000\u0000\u0000\u01c6\u01c7\u0007\u000f"+ + "\u0000\u0000\u01c7\u01c8\u0007\u0006\u0000\u0000\u01c8\u01c9\u0007\u0007"+ + "\u0000\u0000\u01c9\u01ca\u0007\u0010\u0000\u0000\u01ca\u01cb\u0001\u0000"+ + "\u0000\u0000\u01cb\u01cc\u0006\u0005\u0004\u0000\u01cc\u001a\u0001\u0000"+ + "\u0000\u0000\u01cd\u01ce\u0007\u0011\u0000\u0000\u01ce\u01cf\u0007\u0006"+ + "\u0000\u0000\u01cf\u01d0\u0007\u0007\u0000\u0000\u01d0\u01d1\u0007\u0012"+ + "\u0000\u0000\u01d1\u01d2\u0001\u0000\u0000\u0000\u01d2\u01d3\u0006\u0006"+ + "\u0000\u0000\u01d3\u001c\u0001\u0000\u0000\u0000\u01d4\u01d5\u0007\u0012"+ + "\u0000\u0000\u01d5\u01d6\u0007\u0003\u0000\u0000\u01d6\u01d7\u0007\u0003"+ + "\u0000\u0000\u01d7\u01d8\u0007\b\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000"+ + "\u0000\u01d9\u01da\u0006\u0007\u0001\u0000\u01da\u001e\u0001\u0000\u0000"+ + "\u0000\u01db\u01dc\u0007\r\u0000\u0000\u01dc\u01dd\u0007\u0001\u0000\u0000"+ + "\u01dd\u01de\u0007\u0010\u0000\u0000\u01de\u01df\u0007\u0001\u0000\u0000"+ + "\u01df\u01e0\u0007\u0005\u0000\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000"+ + "\u01e1\u01e2\u0006\b\u0000\u0000\u01e2 \u0001\u0000\u0000\u0000\u01e3"+ + "\u01e4\u0007\u0010\u0000\u0000\u01e4\u01e5\u0007\u000b\u0000\u0000\u01e5"+ + "\u01e6\u0005_\u0000\u0000\u01e6\u01e7\u0007\u0003\u0000\u0000\u01e7\u01e8"+ + "\u0007\u000e\u0000\u0000\u01e8\u01e9\u0007\b\u0000\u0000\u01e9\u01ea\u0007"+ + "\f\u0000\u0000\u01ea\u01eb\u0007\t\u0000\u0000\u01eb\u01ec\u0007\u0000"+ + "\u0000\u0000\u01ec\u01ed\u0001\u0000\u0000\u0000\u01ed\u01ee\u0006\t\u0005"+ + "\u0000\u01ee\"\u0001\u0000\u0000\u0000\u01ef\u01f0\u0007\u0006\u0000\u0000"+ + "\u01f0\u01f1\u0007\u0003\u0000\u0000\u01f1\u01f2\u0007\t\u0000\u0000\u01f2"+ + "\u01f3\u0007\f\u0000\u0000\u01f3\u01f4\u0007\u0010\u0000\u0000\u01f4\u01f5"+ + "\u0007\u0003\u0000\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7"+ + "\u0006\n\u0006\u0000\u01f7$\u0001\u0000\u0000\u0000\u01f8\u01f9\u0007"+ + "\u0006\u0000\u0000\u01f9\u01fa\u0007\u0007\u0000\u0000\u01fa\u01fb\u0007"+ + "\u0013\u0000\u0000\u01fb\u01fc\u0001\u0000\u0000\u0000\u01fc\u01fd\u0006"+ + "\u000b\u0000\u0000\u01fd&\u0001\u0000\u0000\u0000\u01fe\u01ff\u0007\u0002"+ + "\u0000\u0000\u01ff\u0200\u0007\n\u0000\u0000\u0200\u0201\u0007\u0007\u0000"+ + "\u0000\u0201\u0202\u0007\u0013\u0000\u0000\u0202\u0203\u0001\u0000\u0000"+ + "\u0000\u0203\u0204\u0006\f\u0007\u0000\u0204(\u0001\u0000\u0000\u0000"+ + "\u0205\u0206\u0007\u0002\u0000\u0000\u0206\u0207\u0007\u0007\u0000\u0000"+ + "\u0207\u0208\u0007\u0006\u0000\u0000\u0208\u0209\u0007\u0005\u0000\u0000"+ + "\u0209\u020a\u0001\u0000\u0000\u0000\u020a\u020b\u0006\r\u0000\u0000\u020b"+ + "*\u0001\u0000\u0000\u0000\u020c\u020d\u0007\u0002\u0000\u0000\u020d\u020e"+ + "\u0007\u0005\u0000\u0000\u020e\u020f\u0007\f\u0000\u0000\u020f\u0210\u0007"+ + "\u0005\u0000\u0000\u0210\u0211\u0007\u0002\u0000\u0000\u0211\u0212\u0001"+ + "\u0000\u0000\u0000\u0212\u0213\u0006\u000e\u0000\u0000\u0213,\u0001\u0000"+ + "\u0000\u0000\u0214\u0215\u0007\u0013\u0000\u0000\u0215\u0216\u0007\n\u0000"+ + "\u0000\u0216\u0217\u0007\u0003\u0000\u0000\u0217\u0218\u0007\u0006\u0000"+ + "\u0000\u0218\u0219\u0007\u0003\u0000\u0000\u0219\u021a\u0001\u0000\u0000"+ + "\u0000\u021a\u021b\u0006\u000f\u0000\u0000\u021b.\u0001\u0000\u0000\u0000"+ + "\u021c\u021d\u0004\u0010\u0000\u0000\u021d\u021e\u0007\u0001\u0000\u0000"+ + "\u021e\u021f\u0007\t\u0000\u0000\u021f\u0220\u0007\r\u0000\u0000\u0220"+ + "\u0221\u0007\u0001\u0000\u0000\u0221\u0222\u0007\t\u0000\u0000\u0222\u0223"+ + "\u0007\u0003\u0000\u0000\u0223\u0224\u0007\u0002\u0000\u0000\u0224\u0225"+ + "\u0007\u0005\u0000\u0000\u0225\u0226\u0007\f\u0000\u0000\u0226\u0227\u0007"+ + "\u0005\u0000\u0000\u0227\u0228\u0007\u0002\u0000\u0000\u0228\u0229\u0001"+ + "\u0000\u0000\u0000\u0229\u022a\u0006\u0010\u0000\u0000\u022a0\u0001\u0000"+ + "\u0000\u0000\u022b\u022c\u0004\u0011\u0001\u0000\u022c\u022d\u0007\r\u0000"+ + "\u0000\u022d\u022e\u0007\u0007\u0000\u0000\u022e\u022f\u0007\u0007\u0000"+ + "\u0000\u022f\u0230\u0007\u0012\u0000\u0000\u0230\u0231\u0007\u0014\u0000"+ + "\u0000\u0231\u0232\u0007\b\u0000\u0000\u0232\u0233\u0001\u0000\u0000\u0000"+ + "\u0233\u0234\u0006\u0011\b\u0000\u02342\u0001\u0000\u0000\u0000\u0235"+ + "\u0236\u0004\u0012\u0002\u0000\u0236\u0237\u0007\u0010\u0000\u0000\u0237"+ + "\u0238\u0007\u0003\u0000\u0000\u0238\u0239\u0007\u0005\u0000\u0000\u0239"+ + "\u023a\u0007\u0006\u0000\u0000\u023a\u023b\u0007\u0001\u0000\u0000\u023b"+ + "\u023c\u0007\u0004\u0000\u0000\u023c\u023d\u0007\u0002\u0000\u0000\u023d"+ + "\u023e\u0001\u0000\u0000\u0000\u023e\u023f\u0006\u0012\t\u0000\u023f4"+ + "\u0001\u0000\u0000\u0000\u0240\u0242\b\u0015\u0000\u0000\u0241\u0240\u0001"+ + "\u0000\u0000\u0000\u0242\u0243\u0001\u0000\u0000\u0000\u0243\u0241\u0001"+ + "\u0000\u0000\u0000\u0243\u0244\u0001\u0000\u0000\u0000\u0244\u0245\u0001"+ + "\u0000\u0000\u0000\u0245\u0246\u0006\u0013\u0000\u0000\u02466\u0001\u0000"+ + "\u0000\u0000\u0247\u0248\u0005/\u0000\u0000\u0248\u0249\u0005/\u0000\u0000"+ + "\u0249\u024d\u0001\u0000\u0000\u0000\u024a\u024c\b\u0016\u0000\u0000\u024b"+ + "\u024a\u0001\u0000\u0000\u0000\u024c\u024f\u0001\u0000\u0000\u0000\u024d"+ + "\u024b\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e"+ + "\u0251\u0001\u0000\u0000\u0000\u024f\u024d\u0001\u0000\u0000\u0000\u0250"+ + "\u0252\u0005\r\u0000\u0000\u0251\u0250\u0001\u0000\u0000\u0000\u0251\u0252"+ + "\u0001\u0000\u0000\u0000\u0252\u0254\u0001\u0000\u0000\u0000\u0253\u0255"+ + "\u0005\n\u0000\u0000\u0254\u0253\u0001\u0000\u0000\u0000\u0254\u0255\u0001"+ + "\u0000\u0000\u0000\u0255\u0256\u0001\u0000\u0000\u0000\u0256\u0257\u0006"+ + "\u0014\n\u0000\u02578\u0001\u0000\u0000\u0000\u0258\u0259\u0005/\u0000"+ + "\u0000\u0259\u025a\u0005*\u0000\u0000\u025a\u025f\u0001\u0000\u0000\u0000"+ + "\u025b\u025e\u00039\u0015\u0000\u025c\u025e\t\u0000\u0000\u0000\u025d"+ + "\u025b\u0001\u0000\u0000\u0000\u025d\u025c\u0001\u0000\u0000\u0000\u025e"+ + "\u0261\u0001\u0000\u0000\u0000\u025f\u0260\u0001\u0000\u0000\u0000\u025f"+ + "\u025d\u0001\u0000\u0000\u0000\u0260\u0262\u0001\u0000\u0000\u0000\u0261"+ + "\u025f\u0001\u0000\u0000\u0000\u0262\u0263\u0005*\u0000\u0000\u0263\u0264"+ + "\u0005/\u0000\u0000\u0264\u0265\u0001\u0000\u0000\u0000\u0265\u0266\u0006"+ + "\u0015\n\u0000\u0266:\u0001\u0000\u0000\u0000\u0267\u0269\u0007\u0017"+ + "\u0000\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0269\u026a\u0001\u0000"+ + "\u0000\u0000\u026a\u0268\u0001\u0000\u0000\u0000\u026a\u026b\u0001\u0000"+ + "\u0000\u0000\u026b\u026c\u0001\u0000\u0000\u0000\u026c\u026d\u0006\u0016"+ + "\n\u0000\u026d<\u0001\u0000\u0000\u0000\u026e\u026f\u0005|\u0000\u0000"+ + "\u026f\u0270\u0001\u0000\u0000\u0000\u0270\u0271\u0006\u0017\u000b\u0000"+ + "\u0271>\u0001\u0000\u0000\u0000\u0272\u0273\u0007\u0018\u0000\u0000\u0273"+ + "@\u0001\u0000\u0000\u0000\u0274\u0275\u0007\u0019\u0000\u0000\u0275B\u0001"+ + "\u0000\u0000\u0000\u0276\u0277\u0005\\\u0000\u0000\u0277\u0278\u0007\u001a"+ + "\u0000\u0000\u0278D\u0001\u0000\u0000\u0000\u0279\u027a\b\u001b\u0000"+ + "\u0000\u027aF\u0001\u0000\u0000\u0000\u027b\u027d\u0007\u0003\u0000\u0000"+ + "\u027c\u027e\u0007\u001c\u0000\u0000\u027d\u027c\u0001\u0000\u0000\u0000"+ + "\u027d\u027e\u0001\u0000\u0000\u0000\u027e\u0280\u0001\u0000\u0000\u0000"+ + "\u027f\u0281\u0003?\u0018\u0000\u0280\u027f\u0001\u0000\u0000\u0000\u0281"+ + "\u0282\u0001\u0000\u0000\u0000\u0282\u0280\u0001\u0000\u0000\u0000\u0282"+ + "\u0283\u0001\u0000\u0000\u0000\u0283H\u0001\u0000\u0000\u0000\u0284\u0285"+ + "\u0005@\u0000\u0000\u0285J\u0001\u0000\u0000\u0000\u0286\u0287\u0005`"+ + "\u0000\u0000\u0287L\u0001\u0000\u0000\u0000\u0288\u028c\b\u001d\u0000"+ + "\u0000\u0289\u028a\u0005`\u0000\u0000\u028a\u028c\u0005`\u0000\u0000\u028b"+ + "\u0288\u0001\u0000\u0000\u0000\u028b\u0289\u0001\u0000\u0000\u0000\u028c"+ + "N\u0001\u0000\u0000\u0000\u028d\u028e\u0005_\u0000\u0000\u028eP\u0001"+ + "\u0000\u0000\u0000\u028f\u0293\u0003A\u0019\u0000\u0290\u0293\u0003?\u0018"+ + "\u0000\u0291\u0293\u0003O \u0000\u0292\u028f\u0001\u0000\u0000\u0000\u0292"+ + "\u0290\u0001\u0000\u0000\u0000\u0292\u0291\u0001\u0000\u0000\u0000\u0293"+ + "R\u0001\u0000\u0000\u0000\u0294\u0299\u0005\"\u0000\u0000\u0295\u0298"+ + "\u0003C\u001a\u0000\u0296\u0298\u0003E\u001b\u0000\u0297\u0295\u0001\u0000"+ + "\u0000\u0000\u0297\u0296\u0001\u0000\u0000\u0000\u0298\u029b\u0001\u0000"+ + "\u0000\u0000\u0299\u0297\u0001\u0000\u0000\u0000\u0299\u029a\u0001\u0000"+ + "\u0000\u0000\u029a\u029c\u0001\u0000\u0000\u0000\u029b\u0299\u0001\u0000"+ + "\u0000\u0000\u029c\u02b2\u0005\"\u0000\u0000\u029d\u029e\u0005\"\u0000"+ + "\u0000\u029e\u029f\u0005\"\u0000\u0000\u029f\u02a0\u0005\"\u0000\u0000"+ + "\u02a0\u02a4\u0001\u0000\u0000\u0000\u02a1\u02a3\b\u0016\u0000\u0000\u02a2"+ + "\u02a1\u0001\u0000\u0000\u0000\u02a3\u02a6\u0001\u0000\u0000\u0000\u02a4"+ + "\u02a5\u0001\u0000\u0000\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5"+ + "\u02a7\u0001\u0000\u0000\u0000\u02a6\u02a4\u0001\u0000\u0000\u0000\u02a7"+ + "\u02a8\u0005\"\u0000\u0000\u02a8\u02a9\u0005\"\u0000\u0000\u02a9\u02aa"+ + "\u0005\"\u0000\u0000\u02aa\u02ac\u0001\u0000\u0000\u0000\u02ab\u02ad\u0005"+ + "\"\u0000\u0000\u02ac\u02ab\u0001\u0000\u0000\u0000\u02ac\u02ad\u0001\u0000"+ + "\u0000\u0000\u02ad\u02af\u0001\u0000\u0000\u0000\u02ae\u02b0\u0005\"\u0000"+ + "\u0000\u02af\u02ae\u0001\u0000\u0000\u0000\u02af\u02b0\u0001\u0000\u0000"+ + "\u0000\u02b0\u02b2\u0001\u0000\u0000\u0000\u02b1\u0294\u0001\u0000\u0000"+ + "\u0000\u02b1\u029d\u0001\u0000\u0000\u0000\u02b2T\u0001\u0000\u0000\u0000"+ + "\u02b3\u02b5\u0003?\u0018\u0000\u02b4\u02b3\u0001\u0000\u0000\u0000\u02b5"+ + "\u02b6\u0001\u0000\u0000\u0000\u02b6\u02b4\u0001\u0000\u0000\u0000\u02b6"+ + "\u02b7\u0001\u0000\u0000\u0000\u02b7V\u0001\u0000\u0000\u0000\u02b8\u02ba"+ + "\u0003?\u0018\u0000\u02b9\u02b8\u0001\u0000\u0000\u0000\u02ba\u02bb\u0001"+ + "\u0000\u0000\u0000\u02bb\u02b9\u0001\u0000\u0000\u0000\u02bb\u02bc\u0001"+ + "\u0000\u0000\u0000\u02bc\u02bd\u0001\u0000\u0000\u0000\u02bd\u02c1\u0003"+ + "g,\u0000\u02be\u02c0\u0003?\u0018\u0000\u02bf\u02be\u0001\u0000\u0000"+ + "\u0000\u02c0\u02c3\u0001\u0000\u0000\u0000\u02c1\u02bf\u0001\u0000\u0000"+ + "\u0000\u02c1\u02c2\u0001\u0000\u0000\u0000\u02c2\u02e3\u0001\u0000\u0000"+ + "\u0000\u02c3\u02c1\u0001\u0000\u0000\u0000\u02c4\u02c6\u0003g,\u0000\u02c5"+ + "\u02c7\u0003?\u0018\u0000\u02c6\u02c5\u0001\u0000\u0000\u0000\u02c7\u02c8"+ + "\u0001\u0000\u0000\u0000\u02c8\u02c6\u0001\u0000\u0000\u0000\u02c8\u02c9"+ + "\u0001\u0000\u0000\u0000\u02c9\u02e3\u0001\u0000\u0000\u0000\u02ca\u02cc"+ + "\u0003?\u0018\u0000\u02cb\u02ca\u0001\u0000\u0000\u0000\u02cc\u02cd\u0001"+ + "\u0000\u0000\u0000\u02cd\u02cb\u0001\u0000\u0000\u0000\u02cd\u02ce\u0001"+ + "\u0000\u0000\u0000\u02ce\u02d6\u0001\u0000\u0000\u0000\u02cf\u02d3\u0003"+ + "g,\u0000\u02d0\u02d2\u0003?\u0018\u0000\u02d1\u02d0\u0001\u0000\u0000"+ + "\u0000\u02d2\u02d5\u0001\u0000\u0000\u0000\u02d3\u02d1\u0001\u0000\u0000"+ + "\u0000\u02d3\u02d4\u0001\u0000\u0000\u0000\u02d4\u02d7\u0001\u0000\u0000"+ + "\u0000\u02d5\u02d3\u0001\u0000\u0000\u0000\u02d6\u02cf\u0001\u0000\u0000"+ + "\u0000\u02d6\u02d7\u0001\u0000\u0000\u0000\u02d7\u02d8\u0001\u0000\u0000"+ + "\u0000\u02d8\u02d9\u0003G\u001c\u0000\u02d9\u02e3\u0001\u0000\u0000\u0000"+ + "\u02da\u02dc\u0003g,\u0000\u02db\u02dd\u0003?\u0018\u0000\u02dc\u02db"+ + "\u0001\u0000\u0000\u0000\u02dd\u02de\u0001\u0000\u0000\u0000\u02de\u02dc"+ + "\u0001\u0000\u0000\u0000\u02de\u02df\u0001\u0000\u0000\u0000\u02df\u02e0"+ + "\u0001\u0000\u0000\u0000\u02e0\u02e1\u0003G\u001c\u0000\u02e1\u02e3\u0001"+ + "\u0000\u0000\u0000\u02e2\u02b9\u0001\u0000\u0000\u0000\u02e2\u02c4\u0001"+ + "\u0000\u0000\u0000\u02e2\u02cb\u0001\u0000\u0000\u0000\u02e2\u02da\u0001"+ + "\u0000\u0000\u0000\u02e3X\u0001\u0000\u0000\u0000\u02e4\u02e5\u0007\u001e"+ + "\u0000\u0000\u02e5\u02e6\u0007\u001f\u0000\u0000\u02e6Z\u0001\u0000\u0000"+ + "\u0000\u02e7\u02e8\u0007\f\u0000\u0000\u02e8\u02e9\u0007\t\u0000\u0000"+ + "\u02e9\u02ea\u0007\u0000\u0000\u0000\u02ea\\\u0001\u0000\u0000\u0000\u02eb"+ + "\u02ec\u0007\f\u0000\u0000\u02ec\u02ed\u0007\u0002\u0000\u0000\u02ed\u02ee"+ + "\u0007\u0004\u0000\u0000\u02ee^\u0001\u0000\u0000\u0000\u02ef\u02f0\u0005"+ + "=\u0000\u0000\u02f0`\u0001\u0000\u0000\u0000\u02f1\u02f2\u0005:\u0000"+ + "\u0000\u02f2\u02f3\u0005:\u0000\u0000\u02f3b\u0001\u0000\u0000\u0000\u02f4"+ + "\u02f5\u0005,\u0000\u0000\u02f5d\u0001\u0000\u0000\u0000\u02f6\u02f7\u0007"+ + "\u0000\u0000\u0000\u02f7\u02f8\u0007\u0003\u0000\u0000\u02f8\u02f9\u0007"+ + "\u0002\u0000\u0000\u02f9\u02fa\u0007\u0004\u0000\u0000\u02faf\u0001\u0000"+ + "\u0000\u0000\u02fb\u02fc\u0005.\u0000\u0000\u02fch\u0001\u0000\u0000\u0000"+ + "\u02fd\u02fe\u0007\u000f\u0000\u0000\u02fe\u02ff\u0007\f\u0000\u0000\u02ff"+ + "\u0300\u0007\r\u0000\u0000\u0300\u0301\u0007\u0002\u0000\u0000\u0301\u0302"+ + "\u0007\u0003\u0000\u0000\u0302j\u0001\u0000\u0000\u0000\u0303\u0304\u0007"+ + "\u000f\u0000\u0000\u0304\u0305\u0007\u0001\u0000\u0000\u0305\u0306\u0007"+ + "\u0006\u0000\u0000\u0306\u0307\u0007\u0002\u0000\u0000\u0307\u0308\u0007"+ + "\u0005\u0000\u0000\u0308l\u0001\u0000\u0000\u0000\u0309\u030a\u0007\u0001"+ + "\u0000\u0000\u030a\u030b\u0007\t\u0000\u0000\u030bn\u0001\u0000\u0000"+ + "\u0000\u030c\u030d\u0007\u0001\u0000\u0000\u030d\u030e\u0007\u0002\u0000"+ + "\u0000\u030ep\u0001\u0000\u0000\u0000\u030f\u0310\u0007\r\u0000\u0000"+ + "\u0310\u0311\u0007\f\u0000\u0000\u0311\u0312\u0007\u0002\u0000\u0000\u0312"+ + "\u0313\u0007\u0005\u0000\u0000\u0313r\u0001\u0000\u0000\u0000\u0314\u0315"+ + "\u0007\r\u0000\u0000\u0315\u0316\u0007\u0001\u0000\u0000\u0316\u0317\u0007"+ + "\u0012\u0000\u0000\u0317\u0318\u0007\u0003\u0000\u0000\u0318t\u0001\u0000"+ + "\u0000\u0000\u0319\u031a\u0005(\u0000\u0000\u031av\u0001\u0000\u0000\u0000"+ + "\u031b\u031c\u0007\t\u0000\u0000\u031c\u031d\u0007\u0007\u0000\u0000\u031d"+ + "\u031e\u0007\u0005\u0000\u0000\u031ex\u0001\u0000\u0000\u0000\u031f\u0320"+ + "\u0007\t\u0000\u0000\u0320\u0321\u0007\u0014\u0000\u0000\u0321\u0322\u0007"+ + "\r\u0000\u0000\u0322\u0323\u0007\r\u0000\u0000\u0323z\u0001\u0000\u0000"+ + "\u0000\u0324\u0325\u0007\t\u0000\u0000\u0325\u0326\u0007\u0014\u0000\u0000"+ + "\u0326\u0327\u0007\r\u0000\u0000\u0327\u0328\u0007\r\u0000\u0000\u0328"+ + "\u0329\u0007\u0002\u0000\u0000\u0329|\u0001\u0000\u0000\u0000\u032a\u032b"+ + "\u0007\u0007\u0000\u0000\u032b\u032c\u0007\u0006\u0000\u0000\u032c~\u0001"+ + "\u0000\u0000\u0000\u032d\u032e\u0005?\u0000\u0000\u032e\u0080\u0001\u0000"+ + "\u0000\u0000\u032f\u0330\u0007\u0006\u0000\u0000\u0330\u0331\u0007\r\u0000"+ + "\u0000\u0331\u0332\u0007\u0001\u0000\u0000\u0332\u0333\u0007\u0012\u0000"+ + "\u0000\u0333\u0334\u0007\u0003\u0000\u0000\u0334\u0082\u0001\u0000\u0000"+ + "\u0000\u0335\u0336\u0005)\u0000\u0000\u0336\u0084\u0001\u0000\u0000\u0000"+ + "\u0337\u0338\u0007\u0005\u0000\u0000\u0338\u0339\u0007\u0006\u0000\u0000"+ + "\u0339\u033a\u0007\u0014\u0000\u0000\u033a\u033b\u0007\u0003\u0000\u0000"+ + "\u033b\u0086\u0001\u0000\u0000\u0000\u033c\u033d\u0005=\u0000\u0000\u033d"+ + "\u033e\u0005=\u0000\u0000\u033e\u0088\u0001\u0000\u0000\u0000\u033f\u0340"+ + "\u0005=\u0000\u0000\u0340\u0341\u0005~\u0000\u0000\u0341\u008a\u0001\u0000"+ + "\u0000\u0000\u0342\u0343\u0005!\u0000\u0000\u0343\u0344\u0005=\u0000\u0000"+ + "\u0344\u008c\u0001\u0000\u0000\u0000\u0345\u0346\u0005<\u0000\u0000\u0346"+ + "\u008e\u0001\u0000\u0000\u0000\u0347\u0348\u0005<\u0000\u0000\u0348\u0349"+ + "\u0005=\u0000\u0000\u0349\u0090\u0001\u0000\u0000\u0000\u034a\u034b\u0005"+ + ">\u0000\u0000\u034b\u0092\u0001\u0000\u0000\u0000\u034c\u034d\u0005>\u0000"+ + "\u0000\u034d\u034e\u0005=\u0000\u0000\u034e\u0094\u0001\u0000\u0000\u0000"+ + "\u034f\u0350\u0005+\u0000\u0000\u0350\u0096\u0001\u0000\u0000\u0000\u0351"+ + "\u0352\u0005-\u0000\u0000\u0352\u0098\u0001\u0000\u0000\u0000\u0353\u0354"+ + "\u0005*\u0000\u0000\u0354\u009a\u0001\u0000\u0000\u0000\u0355\u0356\u0005"+ + "/\u0000\u0000\u0356\u009c\u0001\u0000\u0000\u0000\u0357\u0358\u0005%\u0000"+ + "\u0000\u0358\u009e\u0001\u0000\u0000\u0000\u0359\u035a\u0007\u0010\u0000"+ + "\u0000\u035a\u035b\u0007\f\u0000\u0000\u035b\u035c\u0007\u0005\u0000\u0000"+ + "\u035c\u035d\u0007\u0004\u0000\u0000\u035d\u035e\u0007\n\u0000\u0000\u035e"+ + "\u00a0\u0001\u0000\u0000\u0000\u035f\u0360\u0004I\u0003\u0000\u0360\u0361"+ + "\u0003-\u000f\u0000\u0361\u0362\u0001\u0000\u0000\u0000\u0362\u0363\u0006"+ + "I\f\u0000\u0363\u00a2\u0001\u0000\u0000\u0000\u0364\u0367\u0003\u007f"+ + "8\u0000\u0365\u0368\u0003A\u0019\u0000\u0366\u0368\u0003O \u0000\u0367"+ + "\u0365\u0001\u0000\u0000\u0000\u0367\u0366\u0001\u0000\u0000\u0000\u0368"+ + "\u036c\u0001\u0000\u0000\u0000\u0369\u036b\u0003Q!\u0000\u036a\u0369\u0001"+ + "\u0000\u0000\u0000\u036b\u036e\u0001\u0000\u0000\u0000\u036c\u036a\u0001"+ + "\u0000\u0000\u0000\u036c\u036d\u0001\u0000\u0000\u0000\u036d\u0376\u0001"+ + "\u0000\u0000\u0000\u036e\u036c\u0001\u0000\u0000\u0000\u036f\u0371\u0003"+ + "\u007f8\u0000\u0370\u0372\u0003?\u0018\u0000\u0371\u0370\u0001\u0000\u0000"+ + "\u0000\u0372\u0373\u0001\u0000\u0000\u0000\u0373\u0371\u0001\u0000\u0000"+ + "\u0000\u0373\u0374\u0001\u0000\u0000\u0000\u0374\u0376\u0001\u0000\u0000"+ + "\u0000\u0375\u0364\u0001\u0000\u0000\u0000\u0375\u036f\u0001\u0000\u0000"+ + "\u0000\u0376\u00a4\u0001\u0000\u0000\u0000\u0377\u0378\u0005[\u0000\u0000"+ + "\u0378\u0379\u0001\u0000\u0000\u0000\u0379\u037a\u0006K\u0000\u0000\u037a"+ + "\u037b\u0006K\u0000\u0000\u037b\u00a6\u0001\u0000\u0000\u0000\u037c\u037d"+ + "\u0005]\u0000\u0000\u037d\u037e\u0001\u0000\u0000\u0000\u037e\u037f\u0006"+ + "L\u000b\u0000\u037f\u0380\u0006L\u000b\u0000\u0380\u00a8\u0001\u0000\u0000"+ + "\u0000\u0381\u0385\u0003A\u0019\u0000\u0382\u0384\u0003Q!\u0000\u0383"+ + "\u0382\u0001\u0000\u0000\u0000\u0384\u0387\u0001\u0000\u0000\u0000\u0385"+ + "\u0383\u0001\u0000\u0000\u0000\u0385\u0386\u0001\u0000\u0000\u0000\u0386"+ + "\u0392\u0001\u0000\u0000\u0000\u0387\u0385\u0001\u0000\u0000\u0000\u0388"+ + "\u038b\u0003O \u0000\u0389\u038b\u0003I\u001d\u0000\u038a\u0388\u0001"+ + "\u0000\u0000\u0000\u038a\u0389\u0001\u0000\u0000\u0000\u038b\u038d\u0001"+ + "\u0000\u0000\u0000\u038c\u038e\u0003Q!\u0000\u038d\u038c\u0001\u0000\u0000"+ + "\u0000\u038e\u038f\u0001\u0000\u0000\u0000\u038f\u038d\u0001\u0000\u0000"+ + "\u0000\u038f\u0390\u0001\u0000\u0000\u0000\u0390\u0392\u0001\u0000\u0000"+ + "\u0000\u0391\u0381\u0001\u0000\u0000\u0000\u0391\u038a\u0001\u0000\u0000"+ + "\u0000\u0392\u00aa\u0001\u0000\u0000\u0000\u0393\u0395\u0003K\u001e\u0000"+ + "\u0394\u0396\u0003M\u001f\u0000\u0395\u0394\u0001\u0000\u0000\u0000\u0396"+ + "\u0397\u0001\u0000\u0000\u0000\u0397\u0395\u0001\u0000\u0000\u0000\u0397"+ + "\u0398\u0001\u0000\u0000\u0000\u0398\u0399\u0001\u0000\u0000\u0000\u0399"+ + "\u039a\u0003K\u001e\u0000\u039a\u00ac\u0001\u0000\u0000\u0000\u039b\u039c"+ + "\u0003\u00abN\u0000\u039c\u00ae\u0001\u0000\u0000\u0000\u039d\u039e\u0003"+ + "7\u0014\u0000\u039e\u039f\u0001\u0000\u0000\u0000\u039f\u03a0\u0006P\n"+ + "\u0000\u03a0\u00b0\u0001\u0000\u0000\u0000\u03a1\u03a2\u00039\u0015\u0000"+ + "\u03a2\u03a3\u0001\u0000\u0000\u0000\u03a3\u03a4\u0006Q\n\u0000\u03a4"+ + "\u00b2\u0001\u0000\u0000\u0000\u03a5\u03a6\u0003;\u0016\u0000\u03a6\u03a7"+ + "\u0001\u0000\u0000\u0000\u03a7\u03a8\u0006R\n\u0000\u03a8\u00b4\u0001"+ + "\u0000\u0000\u0000\u03a9\u03aa\u0003\u00a5K\u0000\u03aa\u03ab\u0001\u0000"+ + "\u0000\u0000\u03ab\u03ac\u0006S\r\u0000\u03ac\u03ad\u0006S\u000e\u0000"+ + "\u03ad\u00b6\u0001\u0000\u0000\u0000\u03ae\u03af\u0003=\u0017\u0000\u03af"+ + "\u03b0\u0001\u0000\u0000\u0000\u03b0\u03b1\u0006T\u000f\u0000\u03b1\u03b2"+ + "\u0006T\u000b\u0000\u03b2\u00b8\u0001\u0000\u0000\u0000\u03b3\u03b4\u0003"+ + ";\u0016\u0000\u03b4\u03b5\u0001\u0000\u0000\u0000\u03b5\u03b6\u0006U\n"+ + "\u0000\u03b6\u00ba\u0001\u0000\u0000\u0000\u03b7\u03b8\u00037\u0014\u0000"+ + "\u03b8\u03b9\u0001\u0000\u0000\u0000\u03b9\u03ba\u0006V\n\u0000\u03ba"+ + "\u00bc\u0001\u0000\u0000\u0000\u03bb\u03bc\u00039\u0015\u0000\u03bc\u03bd"+ + "\u0001\u0000\u0000\u0000\u03bd\u03be\u0006W\n\u0000\u03be\u00be\u0001"+ + "\u0000\u0000\u0000\u03bf\u03c0\u0003=\u0017\u0000\u03c0\u03c1\u0001\u0000"+ + "\u0000\u0000\u03c1\u03c2\u0006X\u000f\u0000\u03c2\u03c3\u0006X\u000b\u0000"+ + "\u03c3\u00c0\u0001\u0000\u0000\u0000\u03c4\u03c5\u0003\u00a5K\u0000\u03c5"+ + "\u03c6\u0001\u0000\u0000\u0000\u03c6\u03c7\u0006Y\r\u0000\u03c7\u00c2"+ + "\u0001\u0000\u0000\u0000\u03c8\u03c9\u0003\u00a7L\u0000\u03c9\u03ca\u0001"+ + "\u0000\u0000\u0000\u03ca\u03cb\u0006Z\u0010\u0000\u03cb\u00c4\u0001\u0000"+ + "\u0000\u0000\u03cc\u03cd\u0003\u0151\u00a1\u0000\u03cd\u03ce\u0001\u0000"+ + "\u0000\u0000\u03ce\u03cf\u0006[\u0011\u0000\u03cf\u00c6\u0001\u0000\u0000"+ + "\u0000\u03d0\u03d1\u0003c*\u0000\u03d1\u03d2\u0001\u0000\u0000\u0000\u03d2"+ + "\u03d3\u0006\\\u0012\u0000\u03d3\u00c8\u0001\u0000\u0000\u0000\u03d4\u03d5"+ + "\u0003_(\u0000\u03d5\u03d6\u0001\u0000\u0000\u0000\u03d6\u03d7\u0006]"+ + "\u0013\u0000\u03d7\u00ca\u0001\u0000\u0000\u0000\u03d8\u03d9\u0007\u0010"+ + "\u0000\u0000\u03d9\u03da\u0007\u0003\u0000\u0000\u03da\u03db\u0007\u0005"+ + "\u0000\u0000\u03db\u03dc\u0007\f\u0000\u0000\u03dc\u03dd\u0007\u0000\u0000"+ + "\u0000\u03dd\u03de\u0007\f\u0000\u0000\u03de\u03df\u0007\u0005\u0000\u0000"+ + "\u03df\u03e0\u0007\f\u0000\u0000\u03e0\u00cc\u0001\u0000\u0000\u0000\u03e1"+ + "\u03e5\b \u0000\u0000\u03e2\u03e3\u0005/\u0000\u0000\u03e3\u03e5\b!\u0000"+ + "\u0000\u03e4\u03e1\u0001\u0000\u0000\u0000\u03e4\u03e2\u0001\u0000\u0000"+ + "\u0000\u03e5\u00ce\u0001\u0000\u0000\u0000\u03e6\u03e8\u0003\u00cd_\u0000"+ + "\u03e7\u03e6\u0001\u0000\u0000\u0000\u03e8\u03e9\u0001\u0000\u0000\u0000"+ + "\u03e9\u03e7\u0001\u0000\u0000\u0000\u03e9\u03ea\u0001\u0000\u0000\u0000"+ + "\u03ea\u00d0\u0001\u0000\u0000\u0000\u03eb\u03ec\u0003\u00cf`\u0000\u03ec"+ + "\u03ed\u0001\u0000\u0000\u0000\u03ed\u03ee\u0006a\u0014\u0000\u03ee\u00d2"+ + "\u0001\u0000\u0000\u0000\u03ef\u03f0\u0003S\"\u0000\u03f0\u03f1\u0001"+ + "\u0000\u0000\u0000\u03f1\u03f2\u0006b\u0015\u0000\u03f2\u00d4\u0001\u0000"+ + "\u0000\u0000\u03f3\u03f4\u00037\u0014\u0000\u03f4\u03f5\u0001\u0000\u0000"+ + "\u0000\u03f5\u03f6\u0006c\n\u0000\u03f6\u00d6\u0001\u0000\u0000\u0000"+ + "\u03f7\u03f8\u00039\u0015\u0000\u03f8\u03f9\u0001\u0000\u0000\u0000\u03f9"+ + "\u03fa\u0006d\n\u0000\u03fa\u00d8\u0001\u0000\u0000\u0000\u03fb\u03fc"+ + "\u0003;\u0016\u0000\u03fc\u03fd\u0001\u0000\u0000\u0000\u03fd\u03fe\u0006"+ + "e\n\u0000\u03fe\u00da\u0001\u0000\u0000\u0000\u03ff\u0400\u0003=\u0017"+ + "\u0000\u0400\u0401\u0001\u0000\u0000\u0000\u0401\u0402\u0006f\u000f\u0000"+ + "\u0402\u0403\u0006f\u000b\u0000\u0403\u00dc\u0001\u0000\u0000\u0000\u0404"+ + "\u0405\u0003g,\u0000\u0405\u0406\u0001\u0000\u0000\u0000\u0406\u0407\u0006"+ + "g\u0016\u0000\u0407\u00de\u0001\u0000\u0000\u0000\u0408\u0409\u0003c*"+ + "\u0000\u0409\u040a\u0001\u0000\u0000\u0000\u040a\u040b\u0006h\u0012\u0000"+ + "\u040b\u00e0\u0001\u0000\u0000\u0000\u040c\u040d\u0003\u007f8\u0000\u040d"+ + "\u040e\u0001\u0000\u0000\u0000\u040e\u040f\u0006i\u0017\u0000\u040f\u00e2"+ + "\u0001\u0000\u0000\u0000\u0410\u0411\u0003\u00a3J\u0000\u0411\u0412\u0001"+ + "\u0000\u0000\u0000\u0412\u0413\u0006j\u0018\u0000\u0413\u00e4\u0001\u0000"+ + "\u0000\u0000\u0414\u0419\u0003A\u0019\u0000\u0415\u0419\u0003?\u0018\u0000"+ + "\u0416\u0419\u0003O \u0000\u0417\u0419\u0003\u0099E\u0000\u0418\u0414"+ + "\u0001\u0000\u0000\u0000\u0418\u0415\u0001\u0000\u0000\u0000\u0418\u0416"+ + "\u0001\u0000\u0000\u0000\u0418\u0417\u0001\u0000\u0000\u0000\u0419\u00e6"+ + "\u0001\u0000\u0000\u0000\u041a\u041d\u0003A\u0019\u0000\u041b\u041d\u0003"+ + "\u0099E\u0000\u041c\u041a\u0001\u0000\u0000\u0000\u041c\u041b\u0001\u0000"+ + "\u0000\u0000\u041d\u0421\u0001\u0000\u0000\u0000\u041e\u0420\u0003\u00e5"+ + "k\u0000\u041f\u041e\u0001\u0000\u0000\u0000\u0420\u0423\u0001\u0000\u0000"+ + "\u0000\u0421\u041f\u0001\u0000\u0000\u0000\u0421\u0422\u0001\u0000\u0000"+ + "\u0000\u0422\u042e\u0001\u0000\u0000\u0000\u0423\u0421\u0001\u0000\u0000"+ + "\u0000\u0424\u0427\u0003O \u0000\u0425\u0427\u0003I\u001d\u0000\u0426"+ + "\u0424\u0001\u0000\u0000\u0000\u0426\u0425\u0001\u0000\u0000\u0000\u0427"+ + "\u0429\u0001\u0000\u0000\u0000\u0428\u042a\u0003\u00e5k\u0000\u0429\u0428"+ + "\u0001\u0000\u0000\u0000\u042a\u042b\u0001\u0000\u0000\u0000\u042b\u0429"+ + "\u0001\u0000\u0000\u0000\u042b\u042c\u0001\u0000\u0000\u0000\u042c\u042e"+ + "\u0001\u0000\u0000\u0000\u042d\u041c\u0001\u0000\u0000\u0000\u042d\u0426"+ + "\u0001\u0000\u0000\u0000\u042e\u00e8\u0001\u0000\u0000\u0000\u042f\u0432"+ + "\u0003\u00e7l\u0000\u0430\u0432\u0003\u00abN\u0000\u0431\u042f\u0001\u0000"+ + "\u0000\u0000\u0431\u0430\u0001\u0000\u0000\u0000\u0432\u0433\u0001\u0000"+ + "\u0000\u0000\u0433\u0431\u0001\u0000\u0000\u0000\u0433\u0434\u0001\u0000"+ + "\u0000\u0000\u0434\u00ea\u0001\u0000\u0000\u0000\u0435\u0436\u00037\u0014"+ + "\u0000\u0436\u0437\u0001\u0000\u0000\u0000\u0437\u0438\u0006n\n\u0000"+ + "\u0438\u00ec\u0001\u0000\u0000\u0000\u0439\u043a\u00039\u0015\u0000\u043a"+ + "\u043b\u0001\u0000\u0000\u0000\u043b\u043c\u0006o\n\u0000\u043c\u00ee"+ + "\u0001\u0000\u0000\u0000\u043d\u043e\u0003;\u0016\u0000\u043e\u043f\u0001"+ + "\u0000\u0000\u0000\u043f\u0440\u0006p\n\u0000\u0440\u00f0\u0001\u0000"+ + "\u0000\u0000\u0441\u0442\u0003=\u0017\u0000\u0442\u0443\u0001\u0000\u0000"+ + "\u0000\u0443\u0444\u0006q\u000f\u0000\u0444\u0445\u0006q\u000b\u0000\u0445"+ + "\u00f2\u0001\u0000\u0000\u0000\u0446\u0447\u0003_(\u0000\u0447\u0448\u0001"+ + "\u0000\u0000\u0000\u0448\u0449\u0006r\u0013\u0000\u0449\u00f4\u0001\u0000"+ + "\u0000\u0000\u044a\u044b\u0003c*\u0000\u044b\u044c\u0001\u0000\u0000\u0000"+ + "\u044c\u044d\u0006s\u0012\u0000\u044d\u00f6\u0001\u0000\u0000\u0000\u044e"+ + "\u044f\u0003g,\u0000\u044f\u0450\u0001\u0000\u0000\u0000\u0450\u0451\u0006"+ + "t\u0016\u0000\u0451\u00f8\u0001\u0000\u0000\u0000\u0452\u0453\u0003\u007f"+ + "8\u0000\u0453\u0454\u0001\u0000\u0000\u0000\u0454\u0455\u0006u\u0017\u0000"+ + "\u0455\u00fa\u0001\u0000\u0000\u0000\u0456\u0457\u0003\u00a3J\u0000\u0457"+ + "\u0458\u0001\u0000\u0000\u0000\u0458\u0459\u0006v\u0018\u0000\u0459\u00fc"+ + "\u0001\u0000\u0000\u0000\u045a\u045b\u0007\f\u0000\u0000\u045b\u045c\u0007"+ + "\u0002\u0000\u0000\u045c\u00fe\u0001\u0000\u0000\u0000\u045d\u045e\u0003"+ + "\u00e9m\u0000\u045e\u045f\u0001\u0000\u0000\u0000\u045f\u0460\u0006x\u0019"+ + "\u0000\u0460\u0100\u0001\u0000\u0000\u0000\u0461\u0462\u00037\u0014\u0000"+ + "\u0462\u0463\u0001\u0000\u0000\u0000\u0463\u0464\u0006y\n\u0000\u0464"+ + "\u0102\u0001\u0000\u0000\u0000\u0465\u0466\u00039\u0015\u0000\u0466\u0467"+ + "\u0001\u0000\u0000\u0000\u0467\u0468\u0006z\n\u0000\u0468\u0104\u0001"+ + "\u0000\u0000\u0000\u0469\u046a\u0003;\u0016\u0000\u046a\u046b\u0001\u0000"+ + "\u0000\u0000\u046b\u046c\u0006{\n\u0000\u046c\u0106\u0001\u0000\u0000"+ + "\u0000\u046d\u046e\u0003=\u0017\u0000\u046e\u046f\u0001\u0000\u0000\u0000"+ + "\u046f\u0470\u0006|\u000f\u0000\u0470\u0471\u0006|\u000b\u0000\u0471\u0108"+ + "\u0001\u0000\u0000\u0000\u0472\u0473\u0003\u00a5K\u0000\u0473\u0474\u0001"+ + "\u0000\u0000\u0000\u0474\u0475\u0006}\r\u0000\u0475\u0476\u0006}\u001a"+ + "\u0000\u0476\u010a\u0001\u0000\u0000\u0000\u0477\u0478\u0007\u0007\u0000"+ + "\u0000\u0478\u0479\u0007\t\u0000\u0000\u0479\u047a\u0001\u0000\u0000\u0000"+ + "\u047a\u047b\u0006~\u001b\u0000\u047b\u010c\u0001\u0000\u0000\u0000\u047c"+ + "\u047d\u0007\u0013\u0000\u0000\u047d\u047e\u0007\u0001\u0000\u0000\u047e"+ + "\u047f\u0007\u0005\u0000\u0000\u047f\u0480\u0007\n\u0000\u0000\u0480\u0481"+ + "\u0001\u0000\u0000\u0000\u0481\u0482\u0006\u007f\u001b\u0000\u0482\u010e"+ + "\u0001\u0000\u0000\u0000\u0483\u0484\b\"\u0000\u0000\u0484\u0110\u0001"+ + "\u0000\u0000\u0000\u0485\u0487\u0003\u010f\u0080\u0000\u0486\u0485\u0001"+ + "\u0000\u0000\u0000\u0487\u0488\u0001\u0000\u0000\u0000\u0488\u0486\u0001"+ + "\u0000\u0000\u0000\u0488\u0489\u0001\u0000\u0000\u0000\u0489\u048a\u0001"+ + "\u0000\u0000\u0000\u048a\u048b\u0003\u0151\u00a1\u0000\u048b\u048d\u0001"+ + "\u0000\u0000\u0000\u048c\u0486\u0001\u0000\u0000\u0000\u048c\u048d\u0001"+ + "\u0000\u0000\u0000\u048d\u048f\u0001\u0000\u0000\u0000\u048e\u0490\u0003"+ + "\u010f\u0080\u0000\u048f\u048e\u0001\u0000\u0000\u0000\u0490\u0491\u0001"+ + "\u0000\u0000\u0000\u0491\u048f\u0001\u0000\u0000\u0000\u0491\u0492\u0001"+ + "\u0000\u0000\u0000\u0492\u0112\u0001\u0000\u0000\u0000\u0493\u0494\u0003"+ + "\u0111\u0081\u0000\u0494\u0495\u0001\u0000\u0000\u0000\u0495\u0496\u0006"+ + "\u0082\u001c\u0000\u0496\u0114\u0001\u0000\u0000\u0000\u0497\u0498\u0003"+ + "7\u0014\u0000\u0498\u0499\u0001\u0000\u0000\u0000\u0499\u049a\u0006\u0083"+ + "\n\u0000\u049a\u0116\u0001\u0000\u0000\u0000\u049b\u049c\u00039\u0015"+ + "\u0000\u049c\u049d\u0001\u0000\u0000\u0000\u049d\u049e\u0006\u0084\n\u0000"+ + "\u049e\u0118\u0001\u0000\u0000\u0000\u049f\u04a0\u0003;\u0016\u0000\u04a0"+ + "\u04a1\u0001\u0000\u0000\u0000\u04a1\u04a2\u0006\u0085\n\u0000\u04a2\u011a"+ + "\u0001\u0000\u0000\u0000\u04a3\u04a4\u0003=\u0017\u0000\u04a4\u04a5\u0001"+ + "\u0000\u0000\u0000\u04a5\u04a6\u0006\u0086\u000f\u0000\u04a6\u04a7\u0006"+ + "\u0086\u000b\u0000\u04a7\u04a8\u0006\u0086\u000b\u0000\u04a8\u011c\u0001"+ + "\u0000\u0000\u0000\u04a9\u04aa\u0003_(\u0000\u04aa\u04ab\u0001\u0000\u0000"+ + "\u0000\u04ab\u04ac\u0006\u0087\u0013\u0000\u04ac\u011e\u0001\u0000\u0000"+ + "\u0000\u04ad\u04ae\u0003c*\u0000\u04ae\u04af\u0001\u0000\u0000\u0000\u04af"+ + "\u04b0\u0006\u0088\u0012\u0000\u04b0\u0120\u0001\u0000\u0000\u0000\u04b1"+ + "\u04b2\u0003g,\u0000\u04b2\u04b3\u0001\u0000\u0000\u0000\u04b3\u04b4\u0006"+ + "\u0089\u0016\u0000\u04b4\u0122\u0001\u0000\u0000\u0000\u04b5\u04b6\u0003"+ + "\u010d\u007f\u0000\u04b6\u04b7\u0001\u0000\u0000\u0000\u04b7\u04b8\u0006"+ + "\u008a\u001d\u0000\u04b8\u0124\u0001\u0000\u0000\u0000\u04b9\u04ba\u0003"+ + "\u00e9m\u0000\u04ba\u04bb\u0001\u0000\u0000\u0000\u04bb\u04bc\u0006\u008b"+ + "\u0019\u0000\u04bc\u0126\u0001\u0000\u0000\u0000\u04bd\u04be\u0003\u00ad"+ + "O\u0000\u04be\u04bf\u0001\u0000\u0000\u0000\u04bf\u04c0\u0006\u008c\u001e"+ + "\u0000\u04c0\u0128\u0001\u0000\u0000\u0000\u04c1\u04c2\u0003\u007f8\u0000"+ + "\u04c2\u04c3\u0001\u0000\u0000\u0000\u04c3\u04c4\u0006\u008d\u0017\u0000"+ + "\u04c4\u012a\u0001\u0000\u0000\u0000\u04c5\u04c6\u0003\u00a3J\u0000\u04c6"+ + "\u04c7\u0001\u0000\u0000\u0000\u04c7\u04c8\u0006\u008e\u0018\u0000\u04c8"+ + "\u012c\u0001\u0000\u0000\u0000\u04c9\u04ca\u00037\u0014\u0000\u04ca\u04cb"+ + "\u0001\u0000\u0000\u0000\u04cb\u04cc\u0006\u008f\n\u0000\u04cc\u012e\u0001"+ + "\u0000\u0000\u0000\u04cd\u04ce\u00039\u0015\u0000\u04ce\u04cf\u0001\u0000"+ + "\u0000\u0000\u04cf\u04d0\u0006\u0090\n\u0000\u04d0\u0130\u0001\u0000\u0000"+ + "\u0000\u04d1\u04d2\u0003;\u0016\u0000\u04d2\u04d3\u0001\u0000\u0000\u0000"+ + "\u04d3\u04d4\u0006\u0091\n\u0000\u04d4\u0132\u0001\u0000\u0000\u0000\u04d5"+ + "\u04d6\u0003=\u0017\u0000\u04d6\u04d7\u0001\u0000\u0000\u0000\u04d7\u04d8"+ + "\u0006\u0092\u000f\u0000\u04d8\u04d9\u0006\u0092\u000b\u0000\u04d9\u0134"+ + "\u0001\u0000\u0000\u0000\u04da\u04db\u0003g,\u0000\u04db\u04dc\u0001\u0000"+ + "\u0000\u0000\u04dc\u04dd\u0006\u0093\u0016\u0000\u04dd\u0136\u0001\u0000"+ + "\u0000\u0000\u04de\u04df\u0003\u007f8\u0000\u04df\u04e0\u0001\u0000\u0000"+ + "\u0000\u04e0\u04e1\u0006\u0094\u0017\u0000\u04e1\u0138\u0001\u0000\u0000"+ + "\u0000\u04e2\u04e3\u0003\u00a3J\u0000\u04e3\u04e4\u0001\u0000\u0000\u0000"+ + "\u04e4\u04e5\u0006\u0095\u0018\u0000\u04e5\u013a\u0001\u0000\u0000\u0000"+ + "\u04e6\u04e7\u0003\u00adO\u0000\u04e7\u04e8\u0001\u0000\u0000\u0000\u04e8"+ + "\u04e9\u0006\u0096\u001e\u0000\u04e9\u013c\u0001\u0000\u0000\u0000\u04ea"+ + "\u04eb\u0003\u00a9M\u0000\u04eb\u04ec\u0001\u0000\u0000\u0000\u04ec\u04ed"+ + "\u0006\u0097\u001f\u0000\u04ed\u013e\u0001\u0000\u0000\u0000\u04ee\u04ef"+ + "\u00037\u0014\u0000\u04ef\u04f0\u0001\u0000\u0000\u0000\u04f0\u04f1\u0006"+ + "\u0098\n\u0000\u04f1\u0140\u0001\u0000\u0000\u0000\u04f2\u04f3\u00039"+ + "\u0015\u0000\u04f3\u04f4\u0001\u0000\u0000\u0000\u04f4\u04f5\u0006\u0099"+ + "\n\u0000\u04f5\u0142\u0001\u0000\u0000\u0000\u04f6\u04f7\u0003;\u0016"+ + "\u0000\u04f7\u04f8\u0001\u0000\u0000\u0000\u04f8\u04f9\u0006\u009a\n\u0000"+ + "\u04f9\u0144\u0001\u0000\u0000\u0000\u04fa\u04fb\u0003=\u0017\u0000\u04fb"+ + "\u04fc\u0001\u0000\u0000\u0000\u04fc\u04fd\u0006\u009b\u000f\u0000\u04fd"+ + "\u04fe\u0006\u009b\u000b\u0000\u04fe\u0146\u0001\u0000\u0000\u0000\u04ff"+ + "\u0500\u0007\u0001\u0000\u0000\u0500\u0501\u0007\t\u0000\u0000\u0501\u0502"+ + "\u0007\u000f\u0000\u0000\u0502\u0503\u0007\u0007\u0000\u0000\u0503\u0148"+ + "\u0001\u0000\u0000\u0000\u0504\u0505\u00037\u0014\u0000\u0505\u0506\u0001"+ + "\u0000\u0000\u0000\u0506\u0507\u0006\u009d\n\u0000\u0507\u014a\u0001\u0000"+ + "\u0000\u0000\u0508\u0509\u00039\u0015\u0000\u0509\u050a\u0001\u0000\u0000"+ + "\u0000\u050a\u050b\u0006\u009e\n\u0000\u050b\u014c\u0001\u0000\u0000\u0000"+ + "\u050c\u050d\u0003;\u0016\u0000\u050d\u050e\u0001\u0000\u0000\u0000\u050e"+ + "\u050f\u0006\u009f\n\u0000\u050f\u014e\u0001\u0000\u0000\u0000\u0510\u0511"+ + "\u0003\u00a7L\u0000\u0511\u0512\u0001\u0000\u0000\u0000\u0512\u0513\u0006"+ + "\u00a0\u0010\u0000\u0513\u0514\u0006\u00a0\u000b\u0000\u0514\u0150\u0001"+ + "\u0000\u0000\u0000\u0515\u0516\u0005:\u0000\u0000\u0516\u0152\u0001\u0000"+ + "\u0000\u0000\u0517\u051d\u0003I\u001d\u0000\u0518\u051d\u0003?\u0018\u0000"+ + "\u0519\u051d\u0003g,\u0000\u051a\u051d\u0003A\u0019\u0000\u051b\u051d"+ + "\u0003O \u0000\u051c\u0517\u0001\u0000\u0000\u0000\u051c\u0518\u0001\u0000"+ + "\u0000\u0000\u051c\u0519\u0001\u0000\u0000\u0000\u051c\u051a\u0001\u0000"+ + "\u0000\u0000\u051c\u051b\u0001\u0000\u0000\u0000\u051d\u051e\u0001\u0000"+ + "\u0000\u0000\u051e\u051c\u0001\u0000\u0000\u0000\u051e\u051f\u0001\u0000"+ + "\u0000\u0000\u051f\u0154\u0001\u0000\u0000\u0000\u0520\u0521\u00037\u0014"+ + "\u0000\u0521\u0522\u0001\u0000\u0000\u0000\u0522\u0523\u0006\u00a3\n\u0000"+ + "\u0523\u0156\u0001\u0000\u0000\u0000\u0524\u0525\u00039\u0015\u0000\u0525"+ + "\u0526\u0001\u0000\u0000\u0000\u0526\u0527\u0006\u00a4\n\u0000\u0527\u0158"+ + "\u0001\u0000\u0000\u0000\u0528\u0529\u0003;\u0016\u0000\u0529\u052a\u0001"+ + "\u0000\u0000\u0000\u052a\u052b\u0006\u00a5\n\u0000\u052b\u015a\u0001\u0000"+ + "\u0000\u0000\u052c\u052d\u0003=\u0017\u0000\u052d\u052e\u0001\u0000\u0000"+ + "\u0000\u052e\u052f\u0006\u00a6\u000f\u0000\u052f\u0530\u0006\u00a6\u000b"+ + "\u0000\u0530\u015c\u0001\u0000\u0000\u0000\u0531\u0532\u0003\u0151\u00a1"+ + "\u0000\u0532\u0533\u0001\u0000\u0000\u0000\u0533\u0534\u0006\u00a7\u0011"+ + "\u0000\u0534\u015e\u0001\u0000\u0000\u0000\u0535\u0536\u0003c*\u0000\u0536"+ + "\u0537\u0001\u0000\u0000\u0000\u0537\u0538\u0006\u00a8\u0012\u0000\u0538"+ + "\u0160\u0001\u0000\u0000\u0000\u0539\u053a\u0003g,\u0000\u053a\u053b\u0001"+ + "\u0000\u0000\u0000\u053b\u053c\u0006\u00a9\u0016\u0000\u053c\u0162\u0001"+ + "\u0000\u0000\u0000\u053d\u053e\u0003\u010b~\u0000\u053e\u053f\u0001\u0000"+ + "\u0000\u0000\u053f\u0540\u0006\u00aa \u0000\u0540\u0541\u0006\u00aa!\u0000"+ + "\u0541\u0164\u0001\u0000\u0000\u0000\u0542\u0543\u0003\u00cf`\u0000\u0543"+ + "\u0544\u0001\u0000\u0000\u0000\u0544\u0545\u0006\u00ab\u0014\u0000\u0545"+ + "\u0166\u0001\u0000\u0000\u0000\u0546\u0547\u0003S\"\u0000\u0547\u0548"+ + "\u0001\u0000\u0000\u0000\u0548\u0549\u0006\u00ac\u0015\u0000\u0549\u0168"+ + "\u0001\u0000\u0000\u0000\u054a\u054b\u00037\u0014\u0000\u054b\u054c\u0001"+ + "\u0000\u0000\u0000\u054c\u054d\u0006\u00ad\n\u0000\u054d\u016a\u0001\u0000"+ + "\u0000\u0000\u054e\u054f\u00039\u0015\u0000\u054f\u0550\u0001\u0000\u0000"+ + "\u0000\u0550\u0551\u0006\u00ae\n\u0000\u0551\u016c\u0001\u0000\u0000\u0000"+ + "\u0552\u0553\u0003;\u0016\u0000\u0553\u0554\u0001\u0000\u0000\u0000\u0554"+ + "\u0555\u0006\u00af\n\u0000\u0555\u016e\u0001\u0000\u0000\u0000\u0556\u0557"+ + "\u0003=\u0017\u0000\u0557\u0558\u0001\u0000\u0000\u0000\u0558\u0559\u0006"+ + "\u00b0\u000f\u0000\u0559\u055a\u0006\u00b0\u000b\u0000\u055a\u055b\u0006"+ + "\u00b0\u000b\u0000\u055b\u0170\u0001\u0000\u0000\u0000\u055c\u055d\u0003"+ + "c*\u0000\u055d\u055e\u0001\u0000\u0000\u0000\u055e\u055f\u0006\u00b1\u0012"+ + "\u0000\u055f\u0172\u0001\u0000\u0000\u0000\u0560\u0561\u0003g,\u0000\u0561"+ + "\u0562\u0001\u0000\u0000\u0000\u0562\u0563\u0006\u00b2\u0016\u0000\u0563"+ + "\u0174\u0001\u0000\u0000\u0000\u0564\u0565\u0003\u00e9m\u0000\u0565\u0566"+ + "\u0001\u0000\u0000\u0000\u0566\u0567\u0006\u00b3\u0019\u0000\u0567\u0176"+ + "\u0001\u0000\u0000\u0000\u0568\u0569\u00037\u0014\u0000\u0569\u056a\u0001"+ + "\u0000\u0000\u0000\u056a\u056b\u0006\u00b4\n\u0000\u056b\u0178\u0001\u0000"+ + "\u0000\u0000\u056c\u056d\u00039\u0015\u0000\u056d\u056e\u0001\u0000\u0000"+ + "\u0000\u056e\u056f\u0006\u00b5\n\u0000\u056f\u017a\u0001\u0000\u0000\u0000"+ + "\u0570\u0571\u0003;\u0016\u0000\u0571\u0572\u0001\u0000\u0000\u0000\u0572"+ + "\u0573\u0006\u00b6\n\u0000\u0573\u017c\u0001\u0000\u0000\u0000\u0574\u0575"+ + "\u0003=\u0017\u0000\u0575\u0576\u0001\u0000\u0000\u0000\u0576\u0577\u0006"+ + "\u00b7\u000f\u0000\u0577\u0578\u0006\u00b7\u000b\u0000\u0578\u017e\u0001"+ + "\u0000\u0000\u0000\u0579\u057a\u0003\u00cf`\u0000\u057a\u057b\u0001\u0000"+ + "\u0000\u0000\u057b\u057c\u0006\u00b8\u0014\u0000\u057c\u057d\u0006\u00b8"+ + "\u000b\u0000\u057d\u057e\u0006\u00b8\"\u0000\u057e\u0180\u0001\u0000\u0000"+ + "\u0000\u057f\u0580\u0003S\"\u0000\u0580\u0581\u0001\u0000\u0000\u0000"+ + "\u0581\u0582\u0006\u00b9\u0015\u0000\u0582\u0583\u0006\u00b9\u000b\u0000"+ + "\u0583\u0584\u0006\u00b9\"\u0000\u0584\u0182\u0001\u0000\u0000\u0000\u0585"+ + "\u0586\u00037\u0014\u0000\u0586\u0587\u0001\u0000\u0000\u0000\u0587\u0588"+ + "\u0006\u00ba\n\u0000\u0588\u0184\u0001\u0000\u0000\u0000\u0589\u058a\u0003"+ + "9\u0015\u0000\u058a\u058b\u0001\u0000\u0000\u0000\u058b\u058c\u0006\u00bb"+ + "\n\u0000\u058c\u0186\u0001\u0000\u0000\u0000\u058d\u058e\u0003;\u0016"+ + "\u0000\u058e\u058f\u0001\u0000\u0000\u0000\u058f\u0590\u0006\u00bc\n\u0000"+ + "\u0590\u0188\u0001\u0000\u0000\u0000\u0591\u0592\u0003\u0151\u00a1\u0000"+ + "\u0592\u0593\u0001\u0000\u0000\u0000\u0593\u0594\u0006\u00bd\u0011\u0000"+ + "\u0594\u0595\u0006\u00bd\u000b\u0000\u0595\u0596\u0006\u00bd\t\u0000\u0596"+ + "\u018a\u0001\u0000\u0000\u0000\u0597\u0598\u0003c*\u0000\u0598\u0599\u0001"+ + "\u0000\u0000\u0000\u0599\u059a\u0006\u00be\u0012\u0000\u059a\u059b\u0006"+ + "\u00be\u000b\u0000\u059b\u059c\u0006\u00be\t\u0000\u059c\u018c\u0001\u0000"+ + "\u0000\u0000\u059d\u059e\u00037\u0014\u0000\u059e\u059f\u0001\u0000\u0000"+ + "\u0000\u059f\u05a0\u0006\u00bf\n\u0000\u05a0\u018e\u0001\u0000\u0000\u0000"+ + "\u05a1\u05a2\u00039\u0015\u0000\u05a2\u05a3\u0001\u0000\u0000\u0000\u05a3"+ + "\u05a4\u0006\u00c0\n\u0000\u05a4\u0190\u0001\u0000\u0000\u0000\u05a5\u05a6"+ + "\u0003;\u0016\u0000\u05a6\u05a7\u0001\u0000\u0000\u0000\u05a7\u05a8\u0006"+ + "\u00c1\n\u0000\u05a8\u0192\u0001\u0000\u0000\u0000\u05a9\u05aa\u0003\u00ad"+ + "O\u0000\u05aa\u05ab\u0001\u0000\u0000\u0000\u05ab\u05ac\u0006\u00c2\u000b"+ + "\u0000\u05ac\u05ad\u0006\u00c2\u0000\u0000\u05ad\u05ae\u0006\u00c2\u001e"+ + "\u0000\u05ae\u0194\u0001\u0000\u0000\u0000\u05af\u05b0\u0003\u00a9M\u0000"+ + "\u05b0\u05b1\u0001\u0000\u0000\u0000\u05b1\u05b2\u0006\u00c3\u000b\u0000"+ + "\u05b2\u05b3\u0006\u00c3\u0000\u0000\u05b3\u05b4\u0006\u00c3\u001f\u0000"+ + "\u05b4\u0196\u0001\u0000\u0000\u0000\u05b5\u05b6\u0003Y%\u0000\u05b6\u05b7"+ + "\u0001\u0000\u0000\u0000\u05b7\u05b8\u0006\u00c4\u000b\u0000\u05b8\u05b9"+ + "\u0006\u00c4\u0000\u0000\u05b9\u05ba\u0006\u00c4#\u0000\u05ba\u0198\u0001"+ + "\u0000\u0000\u0000\u05bb\u05bc\u0003=\u0017\u0000\u05bc\u05bd\u0001\u0000"+ + "\u0000\u0000\u05bd\u05be\u0006\u00c5\u000f\u0000\u05be\u05bf\u0006\u00c5"+ + "\u000b\u0000\u05bf\u019a\u0001\u0000\u0000\u0000A\u0000\u0001\u0002\u0003"+ + "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u0243\u024d\u0251\u0254"+ + "\u025d\u025f\u026a\u027d\u0282\u028b\u0292\u0297\u0299\u02a4\u02ac\u02af"+ + "\u02b1\u02b6\u02bb\u02c1\u02c8\u02cd\u02d3\u02d6\u02de\u02e2\u0367\u036c"+ + "\u0373\u0375\u0385\u038a\u038f\u0391\u0397\u03e4\u03e9\u0418\u041c\u0421"+ + "\u0426\u042b\u042d\u0431\u0433\u0488\u048c\u0491\u051c\u051e$\u0005\u0001"+ + "\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005\u0003"+ + "\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b\u0000"+ + "\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0010\u0000\u0007"+ + "A\u0000\u0005\u0000\u0000\u0007\u0018\u0000\u0007B\u0000\u0007h\u0000"+ + "\u0007!\u0000\u0007\u001f\u0000\u0007L\u0000\u0007\u0019\u0000\u0007#"+ + "\u0000\u0007/\u0000\u0007@\u0000\u0007P\u0000\u0005\n\u0000\u0005\u0007"+ "\u0000\u0007Z\u0000\u0007Y\u0000\u0007D\u0000\u0007C\u0000\u0007X\u0000"+ - "\u0005\f\u0000\u0005\u000e\u0000\u0007\u001d\u0000"; + "\u0005\f\u0000\u0005\u000e\u0000\u0007\u001c\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index eb3c70385d628..e718d402982ed 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -23,7 +23,6 @@ null null null null -null '|' null null @@ -63,6 +62,7 @@ null '*' '/' '%' +'match' null null ']' @@ -141,7 +141,6 @@ STATS WHERE DEV_INLINESTATS DEV_LOOKUP -DEV_MATCH DEV_METRICS UNKNOWN_CMD LINE_COMMENT @@ -186,6 +185,7 @@ MINUS ASTERISK SLASH PERCENT +MATCH NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET @@ -257,6 +257,7 @@ valueExpression operatorExpression primaryExpression functionExpression +functionName dataType rowCommand fields @@ -271,6 +272,8 @@ deprecated_metadata metricsCommand evalCommand statsCommand +aggFields +aggField qualifiedName qualifiedNamePattern qualifiedNamePatterns @@ -307,4 +310,4 @@ inlinestatsCommand atn: -[4, 1, 120, 580, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 128, 8, 1, 10, 1, 12, 1, 131, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 139, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 157, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 169, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 176, 8, 5, 10, 5, 12, 5, 179, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 186, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 200, 8, 5, 10, 5, 12, 5, 203, 9, 5, 1, 6, 1, 6, 3, 6, 207, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 214, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 219, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 230, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 236, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 244, 8, 9, 10, 9, 12, 9, 247, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 257, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 262, 8, 10, 10, 10, 12, 10, 265, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 273, 8, 11, 10, 11, 12, 11, 276, 9, 11, 3, 11, 278, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 5, 14, 290, 8, 14, 10, 14, 12, 14, 293, 9, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 300, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 306, 8, 16, 10, 16, 12, 16, 309, 9, 16, 1, 16, 3, 16, 312, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 319, 8, 17, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 3, 20, 327, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 333, 8, 21, 10, 21, 12, 21, 336, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 346, 8, 23, 10, 23, 12, 23, 349, 9, 23, 1, 23, 3, 23, 352, 8, 23, 1, 23, 1, 23, 3, 23, 356, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 3, 25, 363, 8, 25, 1, 25, 1, 25, 3, 25, 367, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 372, 8, 26, 10, 26, 12, 26, 375, 9, 26, 1, 27, 1, 27, 1, 27, 5, 27, 380, 8, 27, 10, 27, 12, 27, 383, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 388, 8, 28, 10, 28, 12, 28, 391, 9, 28, 1, 29, 1, 29, 1, 30, 1, 30, 3, 30, 397, 8, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 412, 8, 31, 10, 31, 12, 31, 415, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 423, 8, 31, 10, 31, 12, 31, 426, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 434, 8, 31, 10, 31, 12, 31, 437, 9, 31, 1, 31, 1, 31, 3, 31, 441, 8, 31, 1, 32, 1, 32, 3, 32, 445, 8, 32, 1, 33, 1, 33, 3, 33, 449, 8, 33, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 458, 8, 35, 10, 35, 12, 35, 461, 9, 35, 1, 36, 1, 36, 3, 36, 465, 8, 36, 1, 36, 1, 36, 3, 36, 469, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 5, 39, 481, 8, 39, 10, 39, 12, 39, 484, 9, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 494, 8, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 5, 44, 506, 8, 44, 10, 44, 12, 44, 509, 9, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 3, 47, 519, 8, 47, 1, 48, 3, 48, 522, 8, 48, 1, 48, 1, 48, 1, 49, 3, 49, 527, 8, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 3, 55, 549, 8, 55, 1, 55, 1, 55, 1, 55, 1, 55, 5, 55, 555, 8, 55, 10, 55, 12, 55, 558, 9, 55, 3, 55, 560, 8, 55, 1, 56, 1, 56, 1, 56, 3, 56, 565, 8, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 578, 8, 58, 1, 58, 0, 4, 2, 10, 18, 20, 59, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 0, 8, 1, 0, 59, 60, 1, 0, 61, 63, 2, 0, 26, 26, 76, 76, 1, 0, 67, 68, 2, 0, 31, 31, 35, 35, 2, 0, 38, 38, 41, 41, 2, 0, 37, 37, 51, 51, 2, 0, 52, 52, 54, 58, 606, 0, 118, 1, 0, 0, 0, 2, 121, 1, 0, 0, 0, 4, 138, 1, 0, 0, 0, 6, 156, 1, 0, 0, 0, 8, 158, 1, 0, 0, 0, 10, 191, 1, 0, 0, 0, 12, 218, 1, 0, 0, 0, 14, 220, 1, 0, 0, 0, 16, 229, 1, 0, 0, 0, 18, 235, 1, 0, 0, 0, 20, 256, 1, 0, 0, 0, 22, 266, 1, 0, 0, 0, 24, 281, 1, 0, 0, 0, 26, 283, 1, 0, 0, 0, 28, 286, 1, 0, 0, 0, 30, 299, 1, 0, 0, 0, 32, 301, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 320, 1, 0, 0, 0, 38, 322, 1, 0, 0, 0, 40, 326, 1, 0, 0, 0, 42, 328, 1, 0, 0, 0, 44, 337, 1, 0, 0, 0, 46, 341, 1, 0, 0, 0, 48, 357, 1, 0, 0, 0, 50, 360, 1, 0, 0, 0, 52, 368, 1, 0, 0, 0, 54, 376, 1, 0, 0, 0, 56, 384, 1, 0, 0, 0, 58, 392, 1, 0, 0, 0, 60, 396, 1, 0, 0, 0, 62, 440, 1, 0, 0, 0, 64, 444, 1, 0, 0, 0, 66, 448, 1, 0, 0, 0, 68, 450, 1, 0, 0, 0, 70, 453, 1, 0, 0, 0, 72, 462, 1, 0, 0, 0, 74, 470, 1, 0, 0, 0, 76, 473, 1, 0, 0, 0, 78, 476, 1, 0, 0, 0, 80, 485, 1, 0, 0, 0, 82, 489, 1, 0, 0, 0, 84, 495, 1, 0, 0, 0, 86, 499, 1, 0, 0, 0, 88, 502, 1, 0, 0, 0, 90, 510, 1, 0, 0, 0, 92, 514, 1, 0, 0, 0, 94, 518, 1, 0, 0, 0, 96, 521, 1, 0, 0, 0, 98, 526, 1, 0, 0, 0, 100, 530, 1, 0, 0, 0, 102, 532, 1, 0, 0, 0, 104, 534, 1, 0, 0, 0, 106, 537, 1, 0, 0, 0, 108, 541, 1, 0, 0, 0, 110, 544, 1, 0, 0, 0, 112, 564, 1, 0, 0, 0, 114, 568, 1, 0, 0, 0, 116, 573, 1, 0, 0, 0, 118, 119, 3, 2, 1, 0, 119, 120, 5, 0, 0, 1, 120, 1, 1, 0, 0, 0, 121, 122, 6, 1, -1, 0, 122, 123, 3, 4, 2, 0, 123, 129, 1, 0, 0, 0, 124, 125, 10, 1, 0, 0, 125, 126, 5, 25, 0, 0, 126, 128, 3, 6, 3, 0, 127, 124, 1, 0, 0, 0, 128, 131, 1, 0, 0, 0, 129, 127, 1, 0, 0, 0, 129, 130, 1, 0, 0, 0, 130, 3, 1, 0, 0, 0, 131, 129, 1, 0, 0, 0, 132, 139, 3, 104, 52, 0, 133, 139, 3, 32, 16, 0, 134, 139, 3, 26, 13, 0, 135, 139, 3, 108, 54, 0, 136, 137, 4, 2, 1, 0, 137, 139, 3, 46, 23, 0, 138, 132, 1, 0, 0, 0, 138, 133, 1, 0, 0, 0, 138, 134, 1, 0, 0, 0, 138, 135, 1, 0, 0, 0, 138, 136, 1, 0, 0, 0, 139, 5, 1, 0, 0, 0, 140, 157, 3, 48, 24, 0, 141, 157, 3, 8, 4, 0, 142, 157, 3, 74, 37, 0, 143, 157, 3, 68, 34, 0, 144, 157, 3, 50, 25, 0, 145, 157, 3, 70, 35, 0, 146, 157, 3, 76, 38, 0, 147, 157, 3, 78, 39, 0, 148, 157, 3, 82, 41, 0, 149, 157, 3, 84, 42, 0, 150, 157, 3, 110, 55, 0, 151, 157, 3, 86, 43, 0, 152, 153, 4, 3, 2, 0, 153, 157, 3, 116, 58, 0, 154, 155, 4, 3, 3, 0, 155, 157, 3, 114, 57, 0, 156, 140, 1, 0, 0, 0, 156, 141, 1, 0, 0, 0, 156, 142, 1, 0, 0, 0, 156, 143, 1, 0, 0, 0, 156, 144, 1, 0, 0, 0, 156, 145, 1, 0, 0, 0, 156, 146, 1, 0, 0, 0, 156, 147, 1, 0, 0, 0, 156, 148, 1, 0, 0, 0, 156, 149, 1, 0, 0, 0, 156, 150, 1, 0, 0, 0, 156, 151, 1, 0, 0, 0, 156, 152, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 157, 7, 1, 0, 0, 0, 158, 159, 5, 16, 0, 0, 159, 160, 3, 10, 5, 0, 160, 9, 1, 0, 0, 0, 161, 162, 6, 5, -1, 0, 162, 163, 5, 44, 0, 0, 163, 192, 3, 10, 5, 8, 164, 192, 3, 16, 8, 0, 165, 192, 3, 12, 6, 0, 166, 168, 3, 16, 8, 0, 167, 169, 5, 44, 0, 0, 168, 167, 1, 0, 0, 0, 168, 169, 1, 0, 0, 0, 169, 170, 1, 0, 0, 0, 170, 171, 5, 39, 0, 0, 171, 172, 5, 43, 0, 0, 172, 177, 3, 16, 8, 0, 173, 174, 5, 34, 0, 0, 174, 176, 3, 16, 8, 0, 175, 173, 1, 0, 0, 0, 176, 179, 1, 0, 0, 0, 177, 175, 1, 0, 0, 0, 177, 178, 1, 0, 0, 0, 178, 180, 1, 0, 0, 0, 179, 177, 1, 0, 0, 0, 180, 181, 5, 50, 0, 0, 181, 192, 1, 0, 0, 0, 182, 183, 3, 16, 8, 0, 183, 185, 5, 40, 0, 0, 184, 186, 5, 44, 0, 0, 185, 184, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 1, 0, 0, 0, 187, 188, 5, 45, 0, 0, 188, 192, 1, 0, 0, 0, 189, 190, 4, 5, 4, 0, 190, 192, 3, 14, 7, 0, 191, 161, 1, 0, 0, 0, 191, 164, 1, 0, 0, 0, 191, 165, 1, 0, 0, 0, 191, 166, 1, 0, 0, 0, 191, 182, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 201, 1, 0, 0, 0, 193, 194, 10, 5, 0, 0, 194, 195, 5, 30, 0, 0, 195, 200, 3, 10, 5, 6, 196, 197, 10, 4, 0, 0, 197, 198, 5, 47, 0, 0, 198, 200, 3, 10, 5, 5, 199, 193, 1, 0, 0, 0, 199, 196, 1, 0, 0, 0, 200, 203, 1, 0, 0, 0, 201, 199, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 11, 1, 0, 0, 0, 203, 201, 1, 0, 0, 0, 204, 206, 3, 16, 8, 0, 205, 207, 5, 44, 0, 0, 206, 205, 1, 0, 0, 0, 206, 207, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 209, 5, 42, 0, 0, 209, 210, 3, 100, 50, 0, 210, 219, 1, 0, 0, 0, 211, 213, 3, 16, 8, 0, 212, 214, 5, 44, 0, 0, 213, 212, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 216, 5, 49, 0, 0, 216, 217, 3, 100, 50, 0, 217, 219, 1, 0, 0, 0, 218, 204, 1, 0, 0, 0, 218, 211, 1, 0, 0, 0, 219, 13, 1, 0, 0, 0, 220, 221, 3, 16, 8, 0, 221, 222, 5, 19, 0, 0, 222, 223, 3, 100, 50, 0, 223, 15, 1, 0, 0, 0, 224, 230, 3, 18, 9, 0, 225, 226, 3, 18, 9, 0, 226, 227, 3, 102, 51, 0, 227, 228, 3, 18, 9, 0, 228, 230, 1, 0, 0, 0, 229, 224, 1, 0, 0, 0, 229, 225, 1, 0, 0, 0, 230, 17, 1, 0, 0, 0, 231, 232, 6, 9, -1, 0, 232, 236, 3, 20, 10, 0, 233, 234, 7, 0, 0, 0, 234, 236, 3, 18, 9, 3, 235, 231, 1, 0, 0, 0, 235, 233, 1, 0, 0, 0, 236, 245, 1, 0, 0, 0, 237, 238, 10, 2, 0, 0, 238, 239, 7, 1, 0, 0, 239, 244, 3, 18, 9, 3, 240, 241, 10, 1, 0, 0, 241, 242, 7, 0, 0, 0, 242, 244, 3, 18, 9, 2, 243, 237, 1, 0, 0, 0, 243, 240, 1, 0, 0, 0, 244, 247, 1, 0, 0, 0, 245, 243, 1, 0, 0, 0, 245, 246, 1, 0, 0, 0, 246, 19, 1, 0, 0, 0, 247, 245, 1, 0, 0, 0, 248, 249, 6, 10, -1, 0, 249, 257, 3, 62, 31, 0, 250, 257, 3, 52, 26, 0, 251, 257, 3, 22, 11, 0, 252, 253, 5, 43, 0, 0, 253, 254, 3, 10, 5, 0, 254, 255, 5, 50, 0, 0, 255, 257, 1, 0, 0, 0, 256, 248, 1, 0, 0, 0, 256, 250, 1, 0, 0, 0, 256, 251, 1, 0, 0, 0, 256, 252, 1, 0, 0, 0, 257, 263, 1, 0, 0, 0, 258, 259, 10, 1, 0, 0, 259, 260, 5, 33, 0, 0, 260, 262, 3, 24, 12, 0, 261, 258, 1, 0, 0, 0, 262, 265, 1, 0, 0, 0, 263, 261, 1, 0, 0, 0, 263, 264, 1, 0, 0, 0, 264, 21, 1, 0, 0, 0, 265, 263, 1, 0, 0, 0, 266, 267, 3, 66, 33, 0, 267, 277, 5, 43, 0, 0, 268, 278, 5, 61, 0, 0, 269, 274, 3, 10, 5, 0, 270, 271, 5, 34, 0, 0, 271, 273, 3, 10, 5, 0, 272, 270, 1, 0, 0, 0, 273, 276, 1, 0, 0, 0, 274, 272, 1, 0, 0, 0, 274, 275, 1, 0, 0, 0, 275, 278, 1, 0, 0, 0, 276, 274, 1, 0, 0, 0, 277, 268, 1, 0, 0, 0, 277, 269, 1, 0, 0, 0, 277, 278, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 280, 5, 50, 0, 0, 280, 23, 1, 0, 0, 0, 281, 282, 3, 58, 29, 0, 282, 25, 1, 0, 0, 0, 283, 284, 5, 12, 0, 0, 284, 285, 3, 28, 14, 0, 285, 27, 1, 0, 0, 0, 286, 291, 3, 30, 15, 0, 287, 288, 5, 34, 0, 0, 288, 290, 3, 30, 15, 0, 289, 287, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 291, 292, 1, 0, 0, 0, 292, 29, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 294, 300, 3, 10, 5, 0, 295, 296, 3, 52, 26, 0, 296, 297, 5, 32, 0, 0, 297, 298, 3, 10, 5, 0, 298, 300, 1, 0, 0, 0, 299, 294, 1, 0, 0, 0, 299, 295, 1, 0, 0, 0, 300, 31, 1, 0, 0, 0, 301, 302, 5, 6, 0, 0, 302, 307, 3, 34, 17, 0, 303, 304, 5, 34, 0, 0, 304, 306, 3, 34, 17, 0, 305, 303, 1, 0, 0, 0, 306, 309, 1, 0, 0, 0, 307, 305, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 311, 1, 0, 0, 0, 309, 307, 1, 0, 0, 0, 310, 312, 3, 40, 20, 0, 311, 310, 1, 0, 0, 0, 311, 312, 1, 0, 0, 0, 312, 33, 1, 0, 0, 0, 313, 314, 3, 36, 18, 0, 314, 315, 5, 104, 0, 0, 315, 316, 3, 38, 19, 0, 316, 319, 1, 0, 0, 0, 317, 319, 3, 38, 19, 0, 318, 313, 1, 0, 0, 0, 318, 317, 1, 0, 0, 0, 319, 35, 1, 0, 0, 0, 320, 321, 5, 76, 0, 0, 321, 37, 1, 0, 0, 0, 322, 323, 7, 2, 0, 0, 323, 39, 1, 0, 0, 0, 324, 327, 3, 42, 21, 0, 325, 327, 3, 44, 22, 0, 326, 324, 1, 0, 0, 0, 326, 325, 1, 0, 0, 0, 327, 41, 1, 0, 0, 0, 328, 329, 5, 75, 0, 0, 329, 334, 5, 76, 0, 0, 330, 331, 5, 34, 0, 0, 331, 333, 5, 76, 0, 0, 332, 330, 1, 0, 0, 0, 333, 336, 1, 0, 0, 0, 334, 332, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 43, 1, 0, 0, 0, 336, 334, 1, 0, 0, 0, 337, 338, 5, 65, 0, 0, 338, 339, 3, 42, 21, 0, 339, 340, 5, 66, 0, 0, 340, 45, 1, 0, 0, 0, 341, 342, 5, 20, 0, 0, 342, 347, 3, 34, 17, 0, 343, 344, 5, 34, 0, 0, 344, 346, 3, 34, 17, 0, 345, 343, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 351, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 350, 352, 3, 28, 14, 0, 351, 350, 1, 0, 0, 0, 351, 352, 1, 0, 0, 0, 352, 355, 1, 0, 0, 0, 353, 354, 5, 29, 0, 0, 354, 356, 3, 28, 14, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 47, 1, 0, 0, 0, 357, 358, 5, 4, 0, 0, 358, 359, 3, 28, 14, 0, 359, 49, 1, 0, 0, 0, 360, 362, 5, 15, 0, 0, 361, 363, 3, 28, 14, 0, 362, 361, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 366, 1, 0, 0, 0, 364, 365, 5, 29, 0, 0, 365, 367, 3, 28, 14, 0, 366, 364, 1, 0, 0, 0, 366, 367, 1, 0, 0, 0, 367, 51, 1, 0, 0, 0, 368, 373, 3, 66, 33, 0, 369, 370, 5, 36, 0, 0, 370, 372, 3, 66, 33, 0, 371, 369, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 371, 1, 0, 0, 0, 373, 374, 1, 0, 0, 0, 374, 53, 1, 0, 0, 0, 375, 373, 1, 0, 0, 0, 376, 381, 3, 60, 30, 0, 377, 378, 5, 36, 0, 0, 378, 380, 3, 60, 30, 0, 379, 377, 1, 0, 0, 0, 380, 383, 1, 0, 0, 0, 381, 379, 1, 0, 0, 0, 381, 382, 1, 0, 0, 0, 382, 55, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 384, 389, 3, 54, 27, 0, 385, 386, 5, 34, 0, 0, 386, 388, 3, 54, 27, 0, 387, 385, 1, 0, 0, 0, 388, 391, 1, 0, 0, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 57, 1, 0, 0, 0, 391, 389, 1, 0, 0, 0, 392, 393, 7, 3, 0, 0, 393, 59, 1, 0, 0, 0, 394, 397, 5, 80, 0, 0, 395, 397, 3, 64, 32, 0, 396, 394, 1, 0, 0, 0, 396, 395, 1, 0, 0, 0, 397, 61, 1, 0, 0, 0, 398, 441, 5, 45, 0, 0, 399, 400, 3, 98, 49, 0, 400, 401, 5, 67, 0, 0, 401, 441, 1, 0, 0, 0, 402, 441, 3, 96, 48, 0, 403, 441, 3, 98, 49, 0, 404, 441, 3, 92, 46, 0, 405, 441, 3, 64, 32, 0, 406, 441, 3, 100, 50, 0, 407, 408, 5, 65, 0, 0, 408, 413, 3, 94, 47, 0, 409, 410, 5, 34, 0, 0, 410, 412, 3, 94, 47, 0, 411, 409, 1, 0, 0, 0, 412, 415, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 413, 414, 1, 0, 0, 0, 414, 416, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 416, 417, 5, 66, 0, 0, 417, 441, 1, 0, 0, 0, 418, 419, 5, 65, 0, 0, 419, 424, 3, 92, 46, 0, 420, 421, 5, 34, 0, 0, 421, 423, 3, 92, 46, 0, 422, 420, 1, 0, 0, 0, 423, 426, 1, 0, 0, 0, 424, 422, 1, 0, 0, 0, 424, 425, 1, 0, 0, 0, 425, 427, 1, 0, 0, 0, 426, 424, 1, 0, 0, 0, 427, 428, 5, 66, 0, 0, 428, 441, 1, 0, 0, 0, 429, 430, 5, 65, 0, 0, 430, 435, 3, 100, 50, 0, 431, 432, 5, 34, 0, 0, 432, 434, 3, 100, 50, 0, 433, 431, 1, 0, 0, 0, 434, 437, 1, 0, 0, 0, 435, 433, 1, 0, 0, 0, 435, 436, 1, 0, 0, 0, 436, 438, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 438, 439, 5, 66, 0, 0, 439, 441, 1, 0, 0, 0, 440, 398, 1, 0, 0, 0, 440, 399, 1, 0, 0, 0, 440, 402, 1, 0, 0, 0, 440, 403, 1, 0, 0, 0, 440, 404, 1, 0, 0, 0, 440, 405, 1, 0, 0, 0, 440, 406, 1, 0, 0, 0, 440, 407, 1, 0, 0, 0, 440, 418, 1, 0, 0, 0, 440, 429, 1, 0, 0, 0, 441, 63, 1, 0, 0, 0, 442, 445, 5, 48, 0, 0, 443, 445, 5, 64, 0, 0, 444, 442, 1, 0, 0, 0, 444, 443, 1, 0, 0, 0, 445, 65, 1, 0, 0, 0, 446, 449, 3, 58, 29, 0, 447, 449, 3, 64, 32, 0, 448, 446, 1, 0, 0, 0, 448, 447, 1, 0, 0, 0, 449, 67, 1, 0, 0, 0, 450, 451, 5, 9, 0, 0, 451, 452, 5, 27, 0, 0, 452, 69, 1, 0, 0, 0, 453, 454, 5, 14, 0, 0, 454, 459, 3, 72, 36, 0, 455, 456, 5, 34, 0, 0, 456, 458, 3, 72, 36, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 71, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 464, 3, 10, 5, 0, 463, 465, 7, 4, 0, 0, 464, 463, 1, 0, 0, 0, 464, 465, 1, 0, 0, 0, 465, 468, 1, 0, 0, 0, 466, 467, 5, 46, 0, 0, 467, 469, 7, 5, 0, 0, 468, 466, 1, 0, 0, 0, 468, 469, 1, 0, 0, 0, 469, 73, 1, 0, 0, 0, 470, 471, 5, 8, 0, 0, 471, 472, 3, 56, 28, 0, 472, 75, 1, 0, 0, 0, 473, 474, 5, 2, 0, 0, 474, 475, 3, 56, 28, 0, 475, 77, 1, 0, 0, 0, 476, 477, 5, 11, 0, 0, 477, 482, 3, 80, 40, 0, 478, 479, 5, 34, 0, 0, 479, 481, 3, 80, 40, 0, 480, 478, 1, 0, 0, 0, 481, 484, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 482, 483, 1, 0, 0, 0, 483, 79, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 486, 3, 54, 27, 0, 486, 487, 5, 84, 0, 0, 487, 488, 3, 54, 27, 0, 488, 81, 1, 0, 0, 0, 489, 490, 5, 1, 0, 0, 490, 491, 3, 20, 10, 0, 491, 493, 3, 100, 50, 0, 492, 494, 3, 88, 44, 0, 493, 492, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 83, 1, 0, 0, 0, 495, 496, 5, 7, 0, 0, 496, 497, 3, 20, 10, 0, 497, 498, 3, 100, 50, 0, 498, 85, 1, 0, 0, 0, 499, 500, 5, 10, 0, 0, 500, 501, 3, 52, 26, 0, 501, 87, 1, 0, 0, 0, 502, 507, 3, 90, 45, 0, 503, 504, 5, 34, 0, 0, 504, 506, 3, 90, 45, 0, 505, 503, 1, 0, 0, 0, 506, 509, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 89, 1, 0, 0, 0, 509, 507, 1, 0, 0, 0, 510, 511, 3, 58, 29, 0, 511, 512, 5, 32, 0, 0, 512, 513, 3, 62, 31, 0, 513, 91, 1, 0, 0, 0, 514, 515, 7, 6, 0, 0, 515, 93, 1, 0, 0, 0, 516, 519, 3, 96, 48, 0, 517, 519, 3, 98, 49, 0, 518, 516, 1, 0, 0, 0, 518, 517, 1, 0, 0, 0, 519, 95, 1, 0, 0, 0, 520, 522, 7, 0, 0, 0, 521, 520, 1, 0, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 1, 0, 0, 0, 523, 524, 5, 28, 0, 0, 524, 97, 1, 0, 0, 0, 525, 527, 7, 0, 0, 0, 526, 525, 1, 0, 0, 0, 526, 527, 1, 0, 0, 0, 527, 528, 1, 0, 0, 0, 528, 529, 5, 27, 0, 0, 529, 99, 1, 0, 0, 0, 530, 531, 5, 26, 0, 0, 531, 101, 1, 0, 0, 0, 532, 533, 7, 7, 0, 0, 533, 103, 1, 0, 0, 0, 534, 535, 5, 5, 0, 0, 535, 536, 3, 106, 53, 0, 536, 105, 1, 0, 0, 0, 537, 538, 5, 65, 0, 0, 538, 539, 3, 2, 1, 0, 539, 540, 5, 66, 0, 0, 540, 107, 1, 0, 0, 0, 541, 542, 5, 13, 0, 0, 542, 543, 5, 100, 0, 0, 543, 109, 1, 0, 0, 0, 544, 545, 5, 3, 0, 0, 545, 548, 5, 90, 0, 0, 546, 547, 5, 88, 0, 0, 547, 549, 3, 54, 27, 0, 548, 546, 1, 0, 0, 0, 548, 549, 1, 0, 0, 0, 549, 559, 1, 0, 0, 0, 550, 551, 5, 89, 0, 0, 551, 556, 3, 112, 56, 0, 552, 553, 5, 34, 0, 0, 553, 555, 3, 112, 56, 0, 554, 552, 1, 0, 0, 0, 555, 558, 1, 0, 0, 0, 556, 554, 1, 0, 0, 0, 556, 557, 1, 0, 0, 0, 557, 560, 1, 0, 0, 0, 558, 556, 1, 0, 0, 0, 559, 550, 1, 0, 0, 0, 559, 560, 1, 0, 0, 0, 560, 111, 1, 0, 0, 0, 561, 562, 3, 54, 27, 0, 562, 563, 5, 32, 0, 0, 563, 565, 1, 0, 0, 0, 564, 561, 1, 0, 0, 0, 564, 565, 1, 0, 0, 0, 565, 566, 1, 0, 0, 0, 566, 567, 3, 54, 27, 0, 567, 113, 1, 0, 0, 0, 568, 569, 5, 18, 0, 0, 569, 570, 3, 34, 17, 0, 570, 571, 5, 88, 0, 0, 571, 572, 3, 56, 28, 0, 572, 115, 1, 0, 0, 0, 573, 574, 5, 17, 0, 0, 574, 577, 3, 28, 14, 0, 575, 576, 5, 29, 0, 0, 576, 578, 3, 28, 14, 0, 577, 575, 1, 0, 0, 0, 577, 578, 1, 0, 0, 0, 578, 117, 1, 0, 0, 0, 56, 129, 138, 156, 168, 177, 185, 191, 199, 201, 206, 213, 218, 229, 235, 243, 245, 256, 263, 274, 277, 291, 299, 307, 311, 318, 326, 334, 347, 351, 355, 362, 366, 373, 381, 389, 396, 413, 424, 435, 440, 444, 448, 459, 464, 468, 482, 493, 507, 518, 521, 526, 548, 556, 559, 564, 577] \ No newline at end of file +[4, 1, 120, 604, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 134, 8, 1, 10, 1, 12, 1, 137, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 145, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 163, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 175, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 182, 8, 5, 10, 5, 12, 5, 185, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 206, 8, 5, 10, 5, 12, 5, 209, 9, 5, 1, 6, 1, 6, 3, 6, 213, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 220, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 236, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 242, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 250, 8, 9, 10, 9, 12, 9, 253, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 263, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 268, 8, 10, 10, 10, 12, 10, 271, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 279, 8, 11, 10, 11, 12, 11, 282, 9, 11, 3, 11, 284, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 3, 12, 290, 8, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 300, 8, 15, 10, 15, 12, 15, 303, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 308, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 316, 8, 17, 10, 17, 12, 17, 319, 9, 17, 1, 17, 3, 17, 322, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 327, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 337, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 343, 8, 22, 10, 22, 12, 22, 346, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 356, 8, 24, 10, 24, 12, 24, 359, 9, 24, 1, 24, 3, 24, 362, 8, 24, 1, 24, 1, 24, 3, 24, 366, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 373, 8, 26, 1, 26, 1, 26, 3, 26, 377, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 382, 8, 27, 10, 27, 12, 27, 385, 9, 27, 1, 28, 1, 28, 1, 28, 1, 28, 3, 28, 391, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 396, 8, 29, 10, 29, 12, 29, 399, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 404, 8, 30, 10, 30, 12, 30, 407, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 412, 8, 31, 10, 31, 12, 31, 415, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 421, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 436, 8, 34, 10, 34, 12, 34, 439, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 3, 34, 465, 8, 34, 1, 35, 1, 35, 3, 35, 469, 8, 35, 1, 36, 1, 36, 3, 36, 473, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 482, 8, 38, 10, 38, 12, 38, 485, 9, 38, 1, 39, 1, 39, 3, 39, 489, 8, 39, 1, 39, 1, 39, 3, 39, 493, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 505, 8, 42, 10, 42, 12, 42, 508, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 518, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 530, 8, 47, 10, 47, 12, 47, 533, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 543, 8, 50, 1, 51, 3, 51, 546, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 551, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 573, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 579, 8, 58, 10, 58, 12, 58, 582, 9, 58, 3, 58, 584, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 589, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 602, 8, 61, 1, 61, 0, 4, 2, 10, 18, 20, 62, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 0, 8, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 25, 25, 76, 76, 1, 0, 67, 68, 2, 0, 30, 30, 34, 34, 2, 0, 37, 37, 40, 40, 2, 0, 36, 36, 50, 50, 2, 0, 51, 51, 53, 57, 630, 0, 124, 1, 0, 0, 0, 2, 127, 1, 0, 0, 0, 4, 144, 1, 0, 0, 0, 6, 162, 1, 0, 0, 0, 8, 164, 1, 0, 0, 0, 10, 197, 1, 0, 0, 0, 12, 224, 1, 0, 0, 0, 14, 226, 1, 0, 0, 0, 16, 235, 1, 0, 0, 0, 18, 241, 1, 0, 0, 0, 20, 262, 1, 0, 0, 0, 22, 272, 1, 0, 0, 0, 24, 289, 1, 0, 0, 0, 26, 291, 1, 0, 0, 0, 28, 293, 1, 0, 0, 0, 30, 296, 1, 0, 0, 0, 32, 307, 1, 0, 0, 0, 34, 311, 1, 0, 0, 0, 36, 326, 1, 0, 0, 0, 38, 330, 1, 0, 0, 0, 40, 332, 1, 0, 0, 0, 42, 336, 1, 0, 0, 0, 44, 338, 1, 0, 0, 0, 46, 347, 1, 0, 0, 0, 48, 351, 1, 0, 0, 0, 50, 367, 1, 0, 0, 0, 52, 370, 1, 0, 0, 0, 54, 378, 1, 0, 0, 0, 56, 386, 1, 0, 0, 0, 58, 392, 1, 0, 0, 0, 60, 400, 1, 0, 0, 0, 62, 408, 1, 0, 0, 0, 64, 416, 1, 0, 0, 0, 66, 420, 1, 0, 0, 0, 68, 464, 1, 0, 0, 0, 70, 468, 1, 0, 0, 0, 72, 472, 1, 0, 0, 0, 74, 474, 1, 0, 0, 0, 76, 477, 1, 0, 0, 0, 78, 486, 1, 0, 0, 0, 80, 494, 1, 0, 0, 0, 82, 497, 1, 0, 0, 0, 84, 500, 1, 0, 0, 0, 86, 509, 1, 0, 0, 0, 88, 513, 1, 0, 0, 0, 90, 519, 1, 0, 0, 0, 92, 523, 1, 0, 0, 0, 94, 526, 1, 0, 0, 0, 96, 534, 1, 0, 0, 0, 98, 538, 1, 0, 0, 0, 100, 542, 1, 0, 0, 0, 102, 545, 1, 0, 0, 0, 104, 550, 1, 0, 0, 0, 106, 554, 1, 0, 0, 0, 108, 556, 1, 0, 0, 0, 110, 558, 1, 0, 0, 0, 112, 561, 1, 0, 0, 0, 114, 565, 1, 0, 0, 0, 116, 568, 1, 0, 0, 0, 118, 588, 1, 0, 0, 0, 120, 592, 1, 0, 0, 0, 122, 597, 1, 0, 0, 0, 124, 125, 3, 2, 1, 0, 125, 126, 5, 0, 0, 1, 126, 1, 1, 0, 0, 0, 127, 128, 6, 1, -1, 0, 128, 129, 3, 4, 2, 0, 129, 135, 1, 0, 0, 0, 130, 131, 10, 1, 0, 0, 131, 132, 5, 24, 0, 0, 132, 134, 3, 6, 3, 0, 133, 130, 1, 0, 0, 0, 134, 137, 1, 0, 0, 0, 135, 133, 1, 0, 0, 0, 135, 136, 1, 0, 0, 0, 136, 3, 1, 0, 0, 0, 137, 135, 1, 0, 0, 0, 138, 145, 3, 110, 55, 0, 139, 145, 3, 34, 17, 0, 140, 145, 3, 28, 14, 0, 141, 145, 3, 114, 57, 0, 142, 143, 4, 2, 1, 0, 143, 145, 3, 48, 24, 0, 144, 138, 1, 0, 0, 0, 144, 139, 1, 0, 0, 0, 144, 140, 1, 0, 0, 0, 144, 141, 1, 0, 0, 0, 144, 142, 1, 0, 0, 0, 145, 5, 1, 0, 0, 0, 146, 163, 3, 50, 25, 0, 147, 163, 3, 8, 4, 0, 148, 163, 3, 80, 40, 0, 149, 163, 3, 74, 37, 0, 150, 163, 3, 52, 26, 0, 151, 163, 3, 76, 38, 0, 152, 163, 3, 82, 41, 0, 153, 163, 3, 84, 42, 0, 154, 163, 3, 88, 44, 0, 155, 163, 3, 90, 45, 0, 156, 163, 3, 116, 58, 0, 157, 163, 3, 92, 46, 0, 158, 159, 4, 3, 2, 0, 159, 163, 3, 122, 61, 0, 160, 161, 4, 3, 3, 0, 161, 163, 3, 120, 60, 0, 162, 146, 1, 0, 0, 0, 162, 147, 1, 0, 0, 0, 162, 148, 1, 0, 0, 0, 162, 149, 1, 0, 0, 0, 162, 150, 1, 0, 0, 0, 162, 151, 1, 0, 0, 0, 162, 152, 1, 0, 0, 0, 162, 153, 1, 0, 0, 0, 162, 154, 1, 0, 0, 0, 162, 155, 1, 0, 0, 0, 162, 156, 1, 0, 0, 0, 162, 157, 1, 0, 0, 0, 162, 158, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 163, 7, 1, 0, 0, 0, 164, 165, 5, 16, 0, 0, 165, 166, 3, 10, 5, 0, 166, 9, 1, 0, 0, 0, 167, 168, 6, 5, -1, 0, 168, 169, 5, 43, 0, 0, 169, 198, 3, 10, 5, 8, 170, 198, 3, 16, 8, 0, 171, 198, 3, 12, 6, 0, 172, 174, 3, 16, 8, 0, 173, 175, 5, 43, 0, 0, 174, 173, 1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 5, 38, 0, 0, 177, 178, 5, 42, 0, 0, 178, 183, 3, 16, 8, 0, 179, 180, 5, 33, 0, 0, 180, 182, 3, 16, 8, 0, 181, 179, 1, 0, 0, 0, 182, 185, 1, 0, 0, 0, 183, 181, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 183, 1, 0, 0, 0, 186, 187, 5, 49, 0, 0, 187, 198, 1, 0, 0, 0, 188, 189, 3, 16, 8, 0, 189, 191, 5, 39, 0, 0, 190, 192, 5, 43, 0, 0, 191, 190, 1, 0, 0, 0, 191, 192, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193, 194, 5, 44, 0, 0, 194, 198, 1, 0, 0, 0, 195, 196, 4, 5, 4, 0, 196, 198, 3, 14, 7, 0, 197, 167, 1, 0, 0, 0, 197, 170, 1, 0, 0, 0, 197, 171, 1, 0, 0, 0, 197, 172, 1, 0, 0, 0, 197, 188, 1, 0, 0, 0, 197, 195, 1, 0, 0, 0, 198, 207, 1, 0, 0, 0, 199, 200, 10, 5, 0, 0, 200, 201, 5, 29, 0, 0, 201, 206, 3, 10, 5, 6, 202, 203, 10, 4, 0, 0, 203, 204, 5, 46, 0, 0, 204, 206, 3, 10, 5, 5, 205, 199, 1, 0, 0, 0, 205, 202, 1, 0, 0, 0, 206, 209, 1, 0, 0, 0, 207, 205, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 11, 1, 0, 0, 0, 209, 207, 1, 0, 0, 0, 210, 212, 3, 16, 8, 0, 211, 213, 5, 43, 0, 0, 212, 211, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 5, 41, 0, 0, 215, 216, 3, 106, 53, 0, 216, 225, 1, 0, 0, 0, 217, 219, 3, 16, 8, 0, 218, 220, 5, 43, 0, 0, 219, 218, 1, 0, 0, 0, 219, 220, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 5, 48, 0, 0, 222, 223, 3, 106, 53, 0, 223, 225, 1, 0, 0, 0, 224, 210, 1, 0, 0, 0, 224, 217, 1, 0, 0, 0, 225, 13, 1, 0, 0, 0, 226, 227, 3, 16, 8, 0, 227, 228, 5, 63, 0, 0, 228, 229, 3, 106, 53, 0, 229, 15, 1, 0, 0, 0, 230, 236, 3, 18, 9, 0, 231, 232, 3, 18, 9, 0, 232, 233, 3, 108, 54, 0, 233, 234, 3, 18, 9, 0, 234, 236, 1, 0, 0, 0, 235, 230, 1, 0, 0, 0, 235, 231, 1, 0, 0, 0, 236, 17, 1, 0, 0, 0, 237, 238, 6, 9, -1, 0, 238, 242, 3, 20, 10, 0, 239, 240, 7, 0, 0, 0, 240, 242, 3, 18, 9, 3, 241, 237, 1, 0, 0, 0, 241, 239, 1, 0, 0, 0, 242, 251, 1, 0, 0, 0, 243, 244, 10, 2, 0, 0, 244, 245, 7, 1, 0, 0, 245, 250, 3, 18, 9, 3, 246, 247, 10, 1, 0, 0, 247, 248, 7, 0, 0, 0, 248, 250, 3, 18, 9, 2, 249, 243, 1, 0, 0, 0, 249, 246, 1, 0, 0, 0, 250, 253, 1, 0, 0, 0, 251, 249, 1, 0, 0, 0, 251, 252, 1, 0, 0, 0, 252, 19, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 254, 255, 6, 10, -1, 0, 255, 263, 3, 68, 34, 0, 256, 263, 3, 58, 29, 0, 257, 263, 3, 22, 11, 0, 258, 259, 5, 42, 0, 0, 259, 260, 3, 10, 5, 0, 260, 261, 5, 49, 0, 0, 261, 263, 1, 0, 0, 0, 262, 254, 1, 0, 0, 0, 262, 256, 1, 0, 0, 0, 262, 257, 1, 0, 0, 0, 262, 258, 1, 0, 0, 0, 263, 269, 1, 0, 0, 0, 264, 265, 10, 1, 0, 0, 265, 266, 5, 32, 0, 0, 266, 268, 3, 26, 13, 0, 267, 264, 1, 0, 0, 0, 268, 271, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 21, 1, 0, 0, 0, 271, 269, 1, 0, 0, 0, 272, 273, 3, 24, 12, 0, 273, 283, 5, 42, 0, 0, 274, 284, 5, 60, 0, 0, 275, 280, 3, 10, 5, 0, 276, 277, 5, 33, 0, 0, 277, 279, 3, 10, 5, 0, 278, 276, 1, 0, 0, 0, 279, 282, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 274, 1, 0, 0, 0, 283, 275, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 285, 1, 0, 0, 0, 285, 286, 5, 49, 0, 0, 286, 23, 1, 0, 0, 0, 287, 290, 5, 63, 0, 0, 288, 290, 3, 72, 36, 0, 289, 287, 1, 0, 0, 0, 289, 288, 1, 0, 0, 0, 290, 25, 1, 0, 0, 0, 291, 292, 3, 64, 32, 0, 292, 27, 1, 0, 0, 0, 293, 294, 5, 12, 0, 0, 294, 295, 3, 30, 15, 0, 295, 29, 1, 0, 0, 0, 296, 301, 3, 32, 16, 0, 297, 298, 5, 33, 0, 0, 298, 300, 3, 32, 16, 0, 299, 297, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 31, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 3, 58, 29, 0, 305, 306, 5, 31, 0, 0, 306, 308, 1, 0, 0, 0, 307, 304, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 310, 3, 10, 5, 0, 310, 33, 1, 0, 0, 0, 311, 312, 5, 6, 0, 0, 312, 317, 3, 36, 18, 0, 313, 314, 5, 33, 0, 0, 314, 316, 3, 36, 18, 0, 315, 313, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 321, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 3, 42, 21, 0, 321, 320, 1, 0, 0, 0, 321, 322, 1, 0, 0, 0, 322, 35, 1, 0, 0, 0, 323, 324, 3, 38, 19, 0, 324, 325, 5, 104, 0, 0, 325, 327, 1, 0, 0, 0, 326, 323, 1, 0, 0, 0, 326, 327, 1, 0, 0, 0, 327, 328, 1, 0, 0, 0, 328, 329, 3, 40, 20, 0, 329, 37, 1, 0, 0, 0, 330, 331, 5, 76, 0, 0, 331, 39, 1, 0, 0, 0, 332, 333, 7, 2, 0, 0, 333, 41, 1, 0, 0, 0, 334, 337, 3, 44, 22, 0, 335, 337, 3, 46, 23, 0, 336, 334, 1, 0, 0, 0, 336, 335, 1, 0, 0, 0, 337, 43, 1, 0, 0, 0, 338, 339, 5, 75, 0, 0, 339, 344, 5, 76, 0, 0, 340, 341, 5, 33, 0, 0, 341, 343, 5, 76, 0, 0, 342, 340, 1, 0, 0, 0, 343, 346, 1, 0, 0, 0, 344, 342, 1, 0, 0, 0, 344, 345, 1, 0, 0, 0, 345, 45, 1, 0, 0, 0, 346, 344, 1, 0, 0, 0, 347, 348, 5, 65, 0, 0, 348, 349, 3, 44, 22, 0, 349, 350, 5, 66, 0, 0, 350, 47, 1, 0, 0, 0, 351, 352, 5, 19, 0, 0, 352, 357, 3, 36, 18, 0, 353, 354, 5, 33, 0, 0, 354, 356, 3, 36, 18, 0, 355, 353, 1, 0, 0, 0, 356, 359, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 361, 1, 0, 0, 0, 359, 357, 1, 0, 0, 0, 360, 362, 3, 54, 27, 0, 361, 360, 1, 0, 0, 0, 361, 362, 1, 0, 0, 0, 362, 365, 1, 0, 0, 0, 363, 364, 5, 28, 0, 0, 364, 366, 3, 30, 15, 0, 365, 363, 1, 0, 0, 0, 365, 366, 1, 0, 0, 0, 366, 49, 1, 0, 0, 0, 367, 368, 5, 4, 0, 0, 368, 369, 3, 30, 15, 0, 369, 51, 1, 0, 0, 0, 370, 372, 5, 15, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 28, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 53, 1, 0, 0, 0, 378, 383, 3, 56, 28, 0, 379, 380, 5, 33, 0, 0, 380, 382, 3, 56, 28, 0, 381, 379, 1, 0, 0, 0, 382, 385, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 55, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 386, 387, 3, 32, 16, 0, 387, 390, 4, 28, 10, 0, 388, 389, 5, 16, 0, 0, 389, 391, 3, 10, 5, 0, 390, 388, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 57, 1, 0, 0, 0, 392, 397, 3, 72, 36, 0, 393, 394, 5, 35, 0, 0, 394, 396, 3, 72, 36, 0, 395, 393, 1, 0, 0, 0, 396, 399, 1, 0, 0, 0, 397, 395, 1, 0, 0, 0, 397, 398, 1, 0, 0, 0, 398, 59, 1, 0, 0, 0, 399, 397, 1, 0, 0, 0, 400, 405, 3, 66, 33, 0, 401, 402, 5, 35, 0, 0, 402, 404, 3, 66, 33, 0, 403, 401, 1, 0, 0, 0, 404, 407, 1, 0, 0, 0, 405, 403, 1, 0, 0, 0, 405, 406, 1, 0, 0, 0, 406, 61, 1, 0, 0, 0, 407, 405, 1, 0, 0, 0, 408, 413, 3, 60, 30, 0, 409, 410, 5, 33, 0, 0, 410, 412, 3, 60, 30, 0, 411, 409, 1, 0, 0, 0, 412, 415, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 413, 414, 1, 0, 0, 0, 414, 63, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 416, 417, 7, 3, 0, 0, 417, 65, 1, 0, 0, 0, 418, 421, 5, 80, 0, 0, 419, 421, 3, 70, 35, 0, 420, 418, 1, 0, 0, 0, 420, 419, 1, 0, 0, 0, 421, 67, 1, 0, 0, 0, 422, 465, 5, 44, 0, 0, 423, 424, 3, 104, 52, 0, 424, 425, 5, 67, 0, 0, 425, 465, 1, 0, 0, 0, 426, 465, 3, 102, 51, 0, 427, 465, 3, 104, 52, 0, 428, 465, 3, 98, 49, 0, 429, 465, 3, 70, 35, 0, 430, 465, 3, 106, 53, 0, 431, 432, 5, 65, 0, 0, 432, 437, 3, 100, 50, 0, 433, 434, 5, 33, 0, 0, 434, 436, 3, 100, 50, 0, 435, 433, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 437, 438, 1, 0, 0, 0, 438, 440, 1, 0, 0, 0, 439, 437, 1, 0, 0, 0, 440, 441, 5, 66, 0, 0, 441, 465, 1, 0, 0, 0, 442, 443, 5, 65, 0, 0, 443, 448, 3, 98, 49, 0, 444, 445, 5, 33, 0, 0, 445, 447, 3, 98, 49, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 66, 0, 0, 452, 465, 1, 0, 0, 0, 453, 454, 5, 65, 0, 0, 454, 459, 3, 106, 53, 0, 455, 456, 5, 33, 0, 0, 456, 458, 3, 106, 53, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 66, 0, 0, 463, 465, 1, 0, 0, 0, 464, 422, 1, 0, 0, 0, 464, 423, 1, 0, 0, 0, 464, 426, 1, 0, 0, 0, 464, 427, 1, 0, 0, 0, 464, 428, 1, 0, 0, 0, 464, 429, 1, 0, 0, 0, 464, 430, 1, 0, 0, 0, 464, 431, 1, 0, 0, 0, 464, 442, 1, 0, 0, 0, 464, 453, 1, 0, 0, 0, 465, 69, 1, 0, 0, 0, 466, 469, 5, 47, 0, 0, 467, 469, 5, 64, 0, 0, 468, 466, 1, 0, 0, 0, 468, 467, 1, 0, 0, 0, 469, 71, 1, 0, 0, 0, 470, 473, 3, 64, 32, 0, 471, 473, 3, 70, 35, 0, 472, 470, 1, 0, 0, 0, 472, 471, 1, 0, 0, 0, 473, 73, 1, 0, 0, 0, 474, 475, 5, 9, 0, 0, 475, 476, 5, 26, 0, 0, 476, 75, 1, 0, 0, 0, 477, 478, 5, 14, 0, 0, 478, 483, 3, 78, 39, 0, 479, 480, 5, 33, 0, 0, 480, 482, 3, 78, 39, 0, 481, 479, 1, 0, 0, 0, 482, 485, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 77, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 486, 488, 3, 10, 5, 0, 487, 489, 7, 4, 0, 0, 488, 487, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 492, 1, 0, 0, 0, 490, 491, 5, 45, 0, 0, 491, 493, 7, 5, 0, 0, 492, 490, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 79, 1, 0, 0, 0, 494, 495, 5, 8, 0, 0, 495, 496, 3, 62, 31, 0, 496, 81, 1, 0, 0, 0, 497, 498, 5, 2, 0, 0, 498, 499, 3, 62, 31, 0, 499, 83, 1, 0, 0, 0, 500, 501, 5, 11, 0, 0, 501, 506, 3, 86, 43, 0, 502, 503, 5, 33, 0, 0, 503, 505, 3, 86, 43, 0, 504, 502, 1, 0, 0, 0, 505, 508, 1, 0, 0, 0, 506, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 85, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 509, 510, 3, 60, 30, 0, 510, 511, 5, 84, 0, 0, 511, 512, 3, 60, 30, 0, 512, 87, 1, 0, 0, 0, 513, 514, 5, 1, 0, 0, 514, 515, 3, 20, 10, 0, 515, 517, 3, 106, 53, 0, 516, 518, 3, 94, 47, 0, 517, 516, 1, 0, 0, 0, 517, 518, 1, 0, 0, 0, 518, 89, 1, 0, 0, 0, 519, 520, 5, 7, 0, 0, 520, 521, 3, 20, 10, 0, 521, 522, 3, 106, 53, 0, 522, 91, 1, 0, 0, 0, 523, 524, 5, 10, 0, 0, 524, 525, 3, 58, 29, 0, 525, 93, 1, 0, 0, 0, 526, 531, 3, 96, 48, 0, 527, 528, 5, 33, 0, 0, 528, 530, 3, 96, 48, 0, 529, 527, 1, 0, 0, 0, 530, 533, 1, 0, 0, 0, 531, 529, 1, 0, 0, 0, 531, 532, 1, 0, 0, 0, 532, 95, 1, 0, 0, 0, 533, 531, 1, 0, 0, 0, 534, 535, 3, 64, 32, 0, 535, 536, 5, 31, 0, 0, 536, 537, 3, 68, 34, 0, 537, 97, 1, 0, 0, 0, 538, 539, 7, 6, 0, 0, 539, 99, 1, 0, 0, 0, 540, 543, 3, 102, 51, 0, 541, 543, 3, 104, 52, 0, 542, 540, 1, 0, 0, 0, 542, 541, 1, 0, 0, 0, 543, 101, 1, 0, 0, 0, 544, 546, 7, 0, 0, 0, 545, 544, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 5, 27, 0, 0, 548, 103, 1, 0, 0, 0, 549, 551, 7, 0, 0, 0, 550, 549, 1, 0, 0, 0, 550, 551, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 553, 5, 26, 0, 0, 553, 105, 1, 0, 0, 0, 554, 555, 5, 25, 0, 0, 555, 107, 1, 0, 0, 0, 556, 557, 7, 7, 0, 0, 557, 109, 1, 0, 0, 0, 558, 559, 5, 5, 0, 0, 559, 560, 3, 112, 56, 0, 560, 111, 1, 0, 0, 0, 561, 562, 5, 65, 0, 0, 562, 563, 3, 2, 1, 0, 563, 564, 5, 66, 0, 0, 564, 113, 1, 0, 0, 0, 565, 566, 5, 13, 0, 0, 566, 567, 5, 100, 0, 0, 567, 115, 1, 0, 0, 0, 568, 569, 5, 3, 0, 0, 569, 572, 5, 90, 0, 0, 570, 571, 5, 88, 0, 0, 571, 573, 3, 60, 30, 0, 572, 570, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 583, 1, 0, 0, 0, 574, 575, 5, 89, 0, 0, 575, 580, 3, 118, 59, 0, 576, 577, 5, 33, 0, 0, 577, 579, 3, 118, 59, 0, 578, 576, 1, 0, 0, 0, 579, 582, 1, 0, 0, 0, 580, 578, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 584, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 583, 574, 1, 0, 0, 0, 583, 584, 1, 0, 0, 0, 584, 117, 1, 0, 0, 0, 585, 586, 3, 60, 30, 0, 586, 587, 5, 31, 0, 0, 587, 589, 1, 0, 0, 0, 588, 585, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 3, 60, 30, 0, 591, 119, 1, 0, 0, 0, 592, 593, 5, 18, 0, 0, 593, 594, 3, 36, 18, 0, 594, 595, 5, 88, 0, 0, 595, 596, 3, 62, 31, 0, 596, 121, 1, 0, 0, 0, 597, 598, 5, 17, 0, 0, 598, 601, 3, 54, 27, 0, 599, 600, 5, 28, 0, 0, 600, 602, 3, 30, 15, 0, 601, 599, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 123, 1, 0, 0, 0, 59, 135, 144, 162, 174, 183, 191, 197, 205, 207, 212, 219, 224, 235, 241, 249, 251, 262, 269, 280, 283, 289, 301, 307, 317, 321, 326, 336, 344, 357, 361, 365, 372, 376, 383, 390, 397, 405, 413, 420, 437, 448, 459, 464, 468, 472, 483, 488, 492, 506, 517, 531, 542, 545, 550, 572, 580, 583, 588, 601] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index 14913849d1b51..ee1ed0a05e978 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -27,13 +27,13 @@ public class EsqlBaseParser extends ParserConfig { public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, - UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, - QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, - ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, - IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, - PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, - GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, + WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, UNKNOWN_CMD=20, + LINE_COMMENT=21, MULTILINE_COMMENT=22, WS=23, PIPE=24, QUOTED_STRING=25, + INTEGER_LITERAL=26, DECIMAL_LITERAL=27, BY=28, AND=29, ASC=30, ASSIGN=31, + CAST_OP=32, COMMA=33, DESC=34, DOT=35, FALSE=36, FIRST=37, IN=38, IS=39, + LAST=40, LIKE=41, LP=42, NOT=43, NULL=44, NULLS=45, OR=46, PARAM=47, RLIKE=48, + RP=49, TRUE=50, EQ=51, CIEQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, MATCH=63, NAMED_OR_POSITIONAL_PARAM=64, OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, @@ -54,36 +54,37 @@ public class EsqlBaseParser extends ParserConfig { RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, RULE_matchBooleanExpression = 7, RULE_valueExpression = 8, RULE_operatorExpression = 9, - RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_dataType = 12, - RULE_rowCommand = 13, RULE_fields = 14, RULE_field = 15, RULE_fromCommand = 16, - RULE_indexPattern = 17, RULE_clusterString = 18, RULE_indexString = 19, - RULE_metadata = 20, RULE_metadataOption = 21, RULE_deprecated_metadata = 22, - RULE_metricsCommand = 23, RULE_evalCommand = 24, RULE_statsCommand = 25, - RULE_qualifiedName = 26, RULE_qualifiedNamePattern = 27, RULE_qualifiedNamePatterns = 28, - RULE_identifier = 29, RULE_identifierPattern = 30, RULE_constant = 31, - RULE_parameter = 32, RULE_identifierOrParameter = 33, RULE_limitCommand = 34, - RULE_sortCommand = 35, RULE_orderExpression = 36, RULE_keepCommand = 37, - RULE_dropCommand = 38, RULE_renameCommand = 39, RULE_renameClause = 40, - RULE_dissectCommand = 41, RULE_grokCommand = 42, RULE_mvExpandCommand = 43, - RULE_commandOptions = 44, RULE_commandOption = 45, RULE_booleanValue = 46, - RULE_numericValue = 47, RULE_decimalValue = 48, RULE_integerValue = 49, - RULE_string = 50, RULE_comparisonOperator = 51, RULE_explainCommand = 52, - RULE_subqueryExpression = 53, RULE_showCommand = 54, RULE_enrichCommand = 55, - RULE_enrichWithClause = 56, RULE_lookupCommand = 57, RULE_inlinestatsCommand = 58; + RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_functionName = 12, + RULE_dataType = 13, RULE_rowCommand = 14, RULE_fields = 15, RULE_field = 16, + RULE_fromCommand = 17, RULE_indexPattern = 18, RULE_clusterString = 19, + RULE_indexString = 20, RULE_metadata = 21, RULE_metadataOption = 22, RULE_deprecated_metadata = 23, + RULE_metricsCommand = 24, RULE_evalCommand = 25, RULE_statsCommand = 26, + RULE_aggFields = 27, RULE_aggField = 28, RULE_qualifiedName = 29, RULE_qualifiedNamePattern = 30, + RULE_qualifiedNamePatterns = 31, RULE_identifier = 32, RULE_identifierPattern = 33, + RULE_constant = 34, RULE_parameter = 35, RULE_identifierOrParameter = 36, + RULE_limitCommand = 37, RULE_sortCommand = 38, RULE_orderExpression = 39, + RULE_keepCommand = 40, RULE_dropCommand = 41, RULE_renameCommand = 42, + RULE_renameClause = 43, RULE_dissectCommand = 44, RULE_grokCommand = 45, + RULE_mvExpandCommand = 46, RULE_commandOptions = 47, RULE_commandOption = 48, + RULE_booleanValue = 49, RULE_numericValue = 50, RULE_decimalValue = 51, + RULE_integerValue = 52, RULE_string = 53, RULE_comparisonOperator = 54, + RULE_explainCommand = 55, RULE_subqueryExpression = 56, RULE_showCommand = 57, + RULE_enrichCommand = 58, RULE_enrichWithClause = 59, RULE_lookupCommand = 60, + RULE_inlinestatsCommand = 61; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", - "dataType", "rowCommand", "fields", "field", "fromCommand", "indexPattern", - "clusterString", "indexString", "metadata", "metadataOption", "deprecated_metadata", - "metricsCommand", "evalCommand", "statsCommand", "qualifiedName", "qualifiedNamePattern", - "qualifiedNamePatterns", "identifier", "identifierPattern", "constant", - "parameter", "identifierOrParameter", "limitCommand", "sortCommand", - "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", - "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", - "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", - "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "functionName", "dataType", "rowCommand", "fields", "field", "fromCommand", + "indexPattern", "clusterString", "indexString", "metadata", "metadataOption", + "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", + "aggFields", "aggField", "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", + "identifier", "identifierPattern", "constant", "parameter", "identifierOrParameter", + "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", + "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", + "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", + "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand" }; @@ -95,15 +96,15 @@ private static String[] makeLiteralNames() { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, - null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", - "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", - "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", - "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", - "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, - null, null, null, null, null, "'metadata'", null, null, null, null, null, - null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, - null, null, null, null, null, null, null, null, "'info'", null, null, - null, "':'" + "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", + "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", + "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", + "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", + "'-'", "'*'", "'/'", "'%'", "'match'", null, null, "']'", null, null, + null, null, null, null, null, null, "'metadata'", null, null, null, null, + null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, + null, null, null, null, null, null, null, null, null, "'info'", null, + null, null, "':'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -111,13 +112,13 @@ private static String[] makeSymbolicNames() { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", - "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", + "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", + "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", + "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", + "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", + "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", + "PERCENT", "MATCH", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", @@ -219,9 +220,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(118); + setState(124); query(0); - setState(119); + setState(125); match(EOF); } } @@ -317,11 +318,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(122); + setState(128); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(129); + setState(135); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -332,16 +333,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(124); + setState(130); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(125); + setState(131); match(PIPE); - setState(126); + setState(132); processingCommand(); } } } - setState(131); + setState(137); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -399,43 +400,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(138); + setState(144); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(132); + setState(138); explainCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(133); + setState(139); fromCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(134); + setState(140); rowCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(135); + setState(141); showCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(136); + setState(142); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(137); + setState(143); metricsCommand(); } break; @@ -520,108 +521,108 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(156); + setState(162); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(140); + setState(146); evalCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(141); + setState(147); whereCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(142); + setState(148); keepCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(143); + setState(149); limitCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(144); + setState(150); statsCommand(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(145); + setState(151); sortCommand(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(146); + setState(152); dropCommand(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(147); + setState(153); renameCommand(); } break; case 9: enterOuterAlt(_localctx, 9); { - setState(148); + setState(154); dissectCommand(); } break; case 10: enterOuterAlt(_localctx, 10); { - setState(149); + setState(155); grokCommand(); } break; case 11: enterOuterAlt(_localctx, 11); { - setState(150); + setState(156); enrichCommand(); } break; case 12: enterOuterAlt(_localctx, 12); { - setState(151); + setState(157); mvExpandCommand(); } break; case 13: enterOuterAlt(_localctx, 13); { - setState(152); + setState(158); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(153); + setState(159); inlinestatsCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(154); + setState(160); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(155); + setState(161); lookupCommand(); } break; @@ -670,9 +671,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(158); + setState(164); match(WHERE); - setState(159); + setState(165); booleanExpression(0); } } @@ -888,7 +889,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(191); + setState(197); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -897,9 +898,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(162); + setState(168); match(NOT); - setState(163); + setState(169); booleanExpression(8); } break; @@ -908,7 +909,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(164); + setState(170); valueExpression(); } break; @@ -917,7 +918,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(165); + setState(171); regexBooleanExpression(); } break; @@ -926,41 +927,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(166); + setState(172); valueExpression(); - setState(168); + setState(174); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(167); + setState(173); match(NOT); } } - setState(170); + setState(176); match(IN); - setState(171); + setState(177); match(LP); - setState(172); + setState(178); valueExpression(); - setState(177); + setState(183); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(173); + setState(179); match(COMMA); - setState(174); + setState(180); valueExpression(); } } - setState(179); + setState(185); _errHandler.sync(this); _la = _input.LA(1); } - setState(180); + setState(186); match(RP); } break; @@ -969,21 +970,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(182); + setState(188); valueExpression(); - setState(183); + setState(189); match(IS); - setState(185); + setState(191); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(184); + setState(190); match(NOT); } } - setState(187); + setState(193); match(NULL); } break; @@ -992,15 +993,15 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(189); + setState(195); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(190); + setState(196); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(201); + setState(207); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1008,7 +1009,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(199); + setState(205); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1016,11 +1017,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(193); + setState(199); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(194); + setState(200); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(195); + setState(201); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1029,18 +1030,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(196); + setState(202); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(197); + setState(203); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(198); + setState(204); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } } } - setState(203); + setState(209); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1095,48 +1096,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(218); + setState(224); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(204); + setState(210); valueExpression(); - setState(206); + setState(212); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(205); + setState(211); match(NOT); } } - setState(208); + setState(214); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(209); + setState(215); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(211); + setState(217); valueExpression(); - setState(213); + setState(219); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(212); + setState(218); match(NOT); } } - setState(215); + setState(221); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(216); + setState(222); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1159,7 +1160,7 @@ public static class MatchBooleanExpressionContext extends ParserRuleContext { public ValueExpressionContext valueExpression() { return getRuleContext(ValueExpressionContext.class,0); } - public TerminalNode DEV_MATCH() { return getToken(EsqlBaseParser.DEV_MATCH, 0); } + public TerminalNode MATCH() { return getToken(EsqlBaseParser.MATCH, 0); } public StringContext string() { return getRuleContext(StringContext.class,0); } @@ -1189,11 +1190,11 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog try { enterOuterAlt(_localctx, 1); { - setState(220); + setState(226); valueExpression(); - setState(221); - match(DEV_MATCH); - setState(222); + setState(227); + match(MATCH); + setState(228); ((MatchBooleanExpressionContext)_localctx).queryString = string(); } } @@ -1277,14 +1278,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(229); + setState(235); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(224); + setState(230); operatorExpression(0); } break; @@ -1292,11 +1293,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(225); + setState(231); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(226); + setState(232); comparisonOperator(); - setState(227); + setState(233); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1421,7 +1422,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(235); + setState(241); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: @@ -1430,7 +1431,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(232); + setState(238); primaryExpression(0); } break; @@ -1439,7 +1440,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(233); + setState(239); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1450,13 +1451,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(234); + setState(240); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(245); + setState(251); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1464,7 +1465,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(243); + setState(249); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1472,12 +1473,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(237); + setState(243); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(238); + setState(244); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -2305843009213693952L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 8070450532247928832L) != 0)) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1485,7 +1486,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(239); + setState(245); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1494,9 +1495,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(240); + setState(246); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(241); + setState(247); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1507,14 +1508,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(242); + setState(248); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(247); + setState(253); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); } @@ -1672,7 +1673,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(256); + setState(262); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { case 1: @@ -1681,7 +1682,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(249); + setState(255); constant(); } break; @@ -1690,7 +1691,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(250); + setState(256); qualifiedName(); } break; @@ -1699,7 +1700,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(251); + setState(257); functionExpression(); } break; @@ -1708,17 +1709,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(252); + setState(258); match(LP); - setState(253); + setState(259); booleanExpression(0); - setState(254); + setState(260); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(263); + setState(269); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,17,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1729,16 +1730,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(258); + setState(264); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(259); + setState(265); match(CAST_OP); - setState(260); + setState(266); dataType(); } } } - setState(265); + setState(271); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,17,_ctx); } @@ -1757,8 +1758,8 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc @SuppressWarnings("CheckReturnValue") public static class FunctionExpressionContext extends ParserRuleContext { - public IdentifierOrParameterContext identifierOrParameter() { - return getRuleContext(IdentifierOrParameterContext.class,0); + public FunctionNameContext functionName() { + return getRuleContext(FunctionNameContext.class,0); } public TerminalNode LP() { return getToken(EsqlBaseParser.LP, 0); } public TerminalNode RP() { return getToken(EsqlBaseParser.RP, 0); } @@ -1800,37 +1801,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(266); - identifierOrParameter(); - setState(267); + setState(272); + functionName(); + setState(273); match(LP); - setState(277); + setState(283); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) { case 1: { - setState(268); + setState(274); match(ASTERISK); } break; case 2: { { - setState(269); + setState(275); booleanExpression(0); - setState(274); + setState(280); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(270); + setState(276); match(COMMA); - setState(271); + setState(277); booleanExpression(0); } } - setState(276); + setState(282); _errHandler.sync(this); _la = _input.LA(1); } @@ -1838,7 +1839,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } break; } - setState(279); + setState(285); match(RP); } } @@ -1853,6 +1854,71 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class FunctionNameContext extends ParserRuleContext { + public TerminalNode MATCH() { return getToken(EsqlBaseParser.MATCH, 0); } + public IdentifierOrParameterContext identifierOrParameter() { + return getRuleContext(IdentifierOrParameterContext.class,0); + } + @SuppressWarnings("this-escape") + public FunctionNameContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_functionName; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterFunctionName(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitFunctionName(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitFunctionName(this); + else return visitor.visitChildren(this); + } + } + + public final FunctionNameContext functionName() throws RecognitionException { + FunctionNameContext _localctx = new FunctionNameContext(_ctx, getState()); + enterRule(_localctx, 24, RULE_functionName); + try { + setState(289); + _errHandler.sync(this); + switch (_input.LA(1)) { + case MATCH: + enterOuterAlt(_localctx, 1); + { + setState(287); + match(MATCH); + } + break; + case PARAM: + case NAMED_OR_POSITIONAL_PARAM: + case UNQUOTED_IDENTIFIER: + case QUOTED_IDENTIFIER: + enterOuterAlt(_localctx, 2); + { + setState(288); + identifierOrParameter(); + } + break; + default: + throw new NoViableAltException(this); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class DataTypeContext extends ParserRuleContext { @SuppressWarnings("this-escape") @@ -1891,12 +1957,12 @@ public T accept(ParseTreeVisitor visitor) { public final DataTypeContext dataType() throws RecognitionException { DataTypeContext _localctx = new DataTypeContext(_ctx, getState()); - enterRule(_localctx, 24, RULE_dataType); + enterRule(_localctx, 26, RULE_dataType); try { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(281); + setState(291); identifier(); } } @@ -1939,13 +2005,13 @@ public T accept(ParseTreeVisitor visitor) { public final RowCommandContext rowCommand() throws RecognitionException { RowCommandContext _localctx = new RowCommandContext(_ctx, getState()); - enterRule(_localctx, 26, RULE_rowCommand); + enterRule(_localctx, 28, RULE_rowCommand); try { enterOuterAlt(_localctx, 1); { - setState(283); + setState(293); match(ROW); - setState(284); + setState(294); fields(); } } @@ -1994,30 +2060,30 @@ public T accept(ParseTreeVisitor visitor) { public final FieldsContext fields() throws RecognitionException { FieldsContext _localctx = new FieldsContext(_ctx, getState()); - enterRule(_localctx, 28, RULE_fields); + enterRule(_localctx, 30, RULE_fields); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(286); + setState(296); field(); - setState(291); + setState(301); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(287); + setState(297); match(COMMA); - setState(288); + setState(298); field(); } } } - setState(293); + setState(303); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } } } @@ -2063,30 +2129,25 @@ public T accept(ParseTreeVisitor visitor) { public final FieldContext field() throws RecognitionException { FieldContext _localctx = new FieldContext(_ctx, getState()); - enterRule(_localctx, 30, RULE_field); + enterRule(_localctx, 32, RULE_field); try { - setState(299); + enterOuterAlt(_localctx, 1); + { + setState(307); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: - enterOuterAlt(_localctx, 1); { - setState(294); - booleanExpression(0); - } - break; - case 2: - enterOuterAlt(_localctx, 2); - { - setState(295); + setState(304); qualifiedName(); - setState(296); + setState(305); match(ASSIGN); - setState(297); - booleanExpression(0); } break; } + setState(309); + booleanExpression(0); + } } catch (RecognitionException re) { _localctx.exception = re; @@ -2137,39 +2198,39 @@ public T accept(ParseTreeVisitor visitor) { public final FromCommandContext fromCommand() throws RecognitionException { FromCommandContext _localctx = new FromCommandContext(_ctx, getState()); - enterRule(_localctx, 32, RULE_fromCommand); + enterRule(_localctx, 34, RULE_fromCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(301); + setState(311); match(FROM); - setState(302); + setState(312); indexPattern(); - setState(307); + setState(317); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(303); + setState(313); match(COMMA); - setState(304); + setState(314); indexPattern(); } } } - setState(309); + setState(319); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } - setState(311); + setState(321); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(310); + setState(320); metadata(); } break; @@ -2189,13 +2250,13 @@ public final FromCommandContext fromCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class IndexPatternContext extends ParserRuleContext { + public IndexStringContext indexString() { + return getRuleContext(IndexStringContext.class,0); + } public ClusterStringContext clusterString() { return getRuleContext(ClusterStringContext.class,0); } public TerminalNode COLON() { return getToken(EsqlBaseParser.COLON, 0); } - public IndexStringContext indexString() { - return getRuleContext(IndexStringContext.class,0); - } @SuppressWarnings("this-escape") public IndexPatternContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -2218,30 +2279,25 @@ public T accept(ParseTreeVisitor visitor) { public final IndexPatternContext indexPattern() throws RecognitionException { IndexPatternContext _localctx = new IndexPatternContext(_ctx, getState()); - enterRule(_localctx, 34, RULE_indexPattern); + enterRule(_localctx, 36, RULE_indexPattern); try { - setState(318); + enterOuterAlt(_localctx, 1); + { + setState(326); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: - enterOuterAlt(_localctx, 1); { - setState(313); + setState(323); clusterString(); - setState(314); + setState(324); match(COLON); - setState(315); - indexString(); - } - break; - case 2: - enterOuterAlt(_localctx, 2); - { - setState(317); - indexString(); } break; } + setState(328); + indexString(); + } } catch (RecognitionException re) { _localctx.exception = re; @@ -2279,11 +2335,11 @@ public T accept(ParseTreeVisitor visitor) { public final ClusterStringContext clusterString() throws RecognitionException { ClusterStringContext _localctx = new ClusterStringContext(_ctx, getState()); - enterRule(_localctx, 36, RULE_clusterString); + enterRule(_localctx, 38, RULE_clusterString); try { enterOuterAlt(_localctx, 1); { - setState(320); + setState(330); match(UNQUOTED_SOURCE); } } @@ -2324,12 +2380,12 @@ public T accept(ParseTreeVisitor visitor) { public final IndexStringContext indexString() throws RecognitionException { IndexStringContext _localctx = new IndexStringContext(_ctx, getState()); - enterRule(_localctx, 38, RULE_indexString); + enterRule(_localctx, 40, RULE_indexString); int _la; try { enterOuterAlt(_localctx, 1); { - setState(322); + setState(332); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2382,22 +2438,22 @@ public T accept(ParseTreeVisitor visitor) { public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); - enterRule(_localctx, 40, RULE_metadata); + enterRule(_localctx, 42, RULE_metadata); try { - setState(326); + setState(336); _errHandler.sync(this); switch (_input.LA(1)) { case METADATA: enterOuterAlt(_localctx, 1); { - setState(324); + setState(334); metadataOption(); } break; case OPENING_BRACKET: enterOuterAlt(_localctx, 2); { - setState(325); + setState(335); deprecated_metadata(); } break; @@ -2449,32 +2505,32 @@ public T accept(ParseTreeVisitor visitor) { public final MetadataOptionContext metadataOption() throws RecognitionException { MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState()); - enterRule(_localctx, 42, RULE_metadataOption); + enterRule(_localctx, 44, RULE_metadataOption); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(328); + setState(338); match(METADATA); - setState(329); + setState(339); match(UNQUOTED_SOURCE); - setState(334); + setState(344); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,26,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(330); + setState(340); match(COMMA); - setState(331); + setState(341); match(UNQUOTED_SOURCE); } } } - setState(336); + setState(346); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,26,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } } } @@ -2517,15 +2573,15 @@ public T accept(ParseTreeVisitor visitor) { public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException { Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_deprecated_metadata); + enterRule(_localctx, 46, RULE_deprecated_metadata); try { enterOuterAlt(_localctx, 1); { - setState(337); + setState(347); match(OPENING_BRACKET); - setState(338); + setState(348); metadataOption(); - setState(339); + setState(349); match(CLOSING_BRACKET); } } @@ -2542,7 +2598,7 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition @SuppressWarnings("CheckReturnValue") public static class MetricsCommandContext extends ParserRuleContext { - public FieldsContext aggregates; + public AggFieldsContext aggregates; public FieldsContext grouping; public TerminalNode DEV_METRICS() { return getToken(EsqlBaseParser.DEV_METRICS, 0); } public List indexPattern() { @@ -2556,11 +2612,11 @@ public TerminalNode COMMA(int i) { return getToken(EsqlBaseParser.COMMA, i); } public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); } - public List fields() { - return getRuleContexts(FieldsContext.class); + public AggFieldsContext aggFields() { + return getRuleContext(AggFieldsContext.class,0); } - public FieldsContext fields(int i) { - return getRuleContext(FieldsContext.class,i); + public FieldsContext fields() { + return getRuleContext(FieldsContext.class,0); } @SuppressWarnings("this-escape") public MetricsCommandContext(ParserRuleContext parent, int invokingState) { @@ -2584,51 +2640,51 @@ public T accept(ParseTreeVisitor visitor) { public final MetricsCommandContext metricsCommand() throws RecognitionException { MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_metricsCommand); + enterRule(_localctx, 48, RULE_metricsCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(341); + setState(351); match(DEV_METRICS); - setState(342); + setState(352); indexPattern(); - setState(347); + setState(357); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,28,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(343); + setState(353); match(COMMA); - setState(344); + setState(354); indexPattern(); } } } - setState(349); + setState(359); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,28,_ctx); } - setState(351); + setState(361); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { - setState(350); - ((MetricsCommandContext)_localctx).aggregates = fields(); + setState(360); + ((MetricsCommandContext)_localctx).aggregates = aggFields(); } break; } - setState(355); + setState(365); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(353); + setState(363); match(BY); - setState(354); + setState(364); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2674,13 +2730,13 @@ public T accept(ParseTreeVisitor visitor) { public final EvalCommandContext evalCommand() throws RecognitionException { EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_evalCommand); + enterRule(_localctx, 50, RULE_evalCommand); try { enterOuterAlt(_localctx, 1); { - setState(357); + setState(367); match(EVAL); - setState(358); + setState(368); fields(); } } @@ -2697,15 +2753,15 @@ public final EvalCommandContext evalCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class StatsCommandContext extends ParserRuleContext { - public FieldsContext stats; + public AggFieldsContext stats; public FieldsContext grouping; public TerminalNode STATS() { return getToken(EsqlBaseParser.STATS, 0); } public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); } - public List fields() { - return getRuleContexts(FieldsContext.class); + public AggFieldsContext aggFields() { + return getRuleContext(AggFieldsContext.class,0); } - public FieldsContext fields(int i) { - return getRuleContext(FieldsContext.class,i); + public FieldsContext fields() { + return getRuleContext(FieldsContext.class,0); } @SuppressWarnings("this-escape") public StatsCommandContext(ParserRuleContext parent, int invokingState) { @@ -2729,30 +2785,30 @@ public T accept(ParseTreeVisitor visitor) { public final StatsCommandContext statsCommand() throws RecognitionException { StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState()); - enterRule(_localctx, 50, RULE_statsCommand); + enterRule(_localctx, 52, RULE_statsCommand); try { enterOuterAlt(_localctx, 1); { - setState(360); + setState(370); match(STATS); - setState(362); + setState(372); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(361); - ((StatsCommandContext)_localctx).stats = fields(); + setState(371); + ((StatsCommandContext)_localctx).stats = aggFields(); } break; } - setState(366); + setState(376); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { case 1: { - setState(364); + setState(374); match(BY); - setState(365); + setState(375); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2770,6 +2826,142 @@ public final StatsCommandContext statsCommand() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class AggFieldsContext extends ParserRuleContext { + public List aggField() { + return getRuleContexts(AggFieldContext.class); + } + public AggFieldContext aggField(int i) { + return getRuleContext(AggFieldContext.class,i); + } + public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(EsqlBaseParser.COMMA, i); + } + @SuppressWarnings("this-escape") + public AggFieldsContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_aggFields; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterAggFields(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitAggFields(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitAggFields(this); + else return visitor.visitChildren(this); + } + } + + public final AggFieldsContext aggFields() throws RecognitionException { + AggFieldsContext _localctx = new AggFieldsContext(_ctx, getState()); + enterRule(_localctx, 54, RULE_aggFields); + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + setState(378); + aggField(); + setState(383); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + { + { + setState(379); + match(COMMA); + setState(380); + aggField(); + } + } + } + setState(385); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + @SuppressWarnings("CheckReturnValue") + public static class AggFieldContext extends ParserRuleContext { + public FieldContext field() { + return getRuleContext(FieldContext.class,0); + } + public TerminalNode WHERE() { return getToken(EsqlBaseParser.WHERE, 0); } + public BooleanExpressionContext booleanExpression() { + return getRuleContext(BooleanExpressionContext.class,0); + } + @SuppressWarnings("this-escape") + public AggFieldContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_aggField; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterAggField(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitAggField(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitAggField(this); + else return visitor.visitChildren(this); + } + } + + public final AggFieldContext aggField() throws RecognitionException { + AggFieldContext _localctx = new AggFieldContext(_ctx, getState()); + enterRule(_localctx, 56, RULE_aggField); + try { + enterOuterAlt(_localctx, 1); + { + setState(386); + field(); + setState(387); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); + setState(390); + _errHandler.sync(this); + switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + case 1: + { + setState(388); + match(WHERE); + setState(389); + booleanExpression(0); + } + break; + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class QualifiedNameContext extends ParserRuleContext { public List identifierOrParameter() { @@ -2804,30 +2996,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNameContext qualifiedName() throws RecognitionException { QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_qualifiedName); + enterRule(_localctx, 58, RULE_qualifiedName); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(368); + setState(392); identifierOrParameter(); - setState(373); + setState(397); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,32,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(369); + setState(393); match(DOT); - setState(370); + setState(394); identifierOrParameter(); } } } - setState(375); + setState(399); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,32,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); } } } @@ -2876,30 +3068,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException { QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_qualifiedNamePattern); + enterRule(_localctx, 60, RULE_qualifiedNamePattern); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(376); + setState(400); identifierPattern(); - setState(381); + setState(405); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(377); + setState(401); match(DOT); - setState(378); + setState(402); identifierPattern(); } } } - setState(383); + setState(407); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); } } } @@ -2948,30 +3140,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException { QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_qualifiedNamePatterns); + enterRule(_localctx, 62, RULE_qualifiedNamePatterns); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(384); + setState(408); qualifiedNamePattern(); - setState(389); + setState(413); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,34,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(385); + setState(409); match(COMMA); - setState(386); + setState(410); qualifiedNamePattern(); } } } - setState(391); + setState(415); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,34,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); } } } @@ -3012,12 +3204,12 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_identifier); + enterRule(_localctx, 64, RULE_identifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(392); + setState(416); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3068,15 +3260,15 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierPatternContext identifierPattern() throws RecognitionException { IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_identifierPattern); + enterRule(_localctx, 66, RULE_identifierPattern); try { - setState(396); + setState(420); _errHandler.sync(this); switch (_input.LA(1)) { case ID_PATTERN: enterOuterAlt(_localctx, 1); { - setState(394); + setState(418); match(ID_PATTERN); } break; @@ -3084,7 +3276,7 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(395); + setState(419); parameter(); } break; @@ -3356,17 +3548,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_constant); + enterRule(_localctx, 68, RULE_constant); int _la; try { - setState(440); + setState(464); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(398); + setState(422); match(NULL); } break; @@ -3374,9 +3566,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(399); + setState(423); integerValue(); - setState(400); + setState(424); match(UNQUOTED_IDENTIFIER); } break; @@ -3384,7 +3576,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(402); + setState(426); decimalValue(); } break; @@ -3392,7 +3584,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(403); + setState(427); integerValue(); } break; @@ -3400,7 +3592,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(404); + setState(428); booleanValue(); } break; @@ -3408,7 +3600,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(405); + setState(429); parameter(); } break; @@ -3416,7 +3608,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(406); + setState(430); string(); } break; @@ -3424,27 +3616,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(407); + setState(431); match(OPENING_BRACKET); - setState(408); + setState(432); numericValue(); - setState(413); + setState(437); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(409); + setState(433); match(COMMA); - setState(410); + setState(434); numericValue(); } } - setState(415); + setState(439); _errHandler.sync(this); _la = _input.LA(1); } - setState(416); + setState(440); match(CLOSING_BRACKET); } break; @@ -3452,27 +3644,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(418); + setState(442); match(OPENING_BRACKET); - setState(419); + setState(443); booleanValue(); - setState(424); + setState(448); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(420); + setState(444); match(COMMA); - setState(421); + setState(445); booleanValue(); } } - setState(426); + setState(450); _errHandler.sync(this); _la = _input.LA(1); } - setState(427); + setState(451); match(CLOSING_BRACKET); } break; @@ -3480,27 +3672,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(429); + setState(453); match(OPENING_BRACKET); - setState(430); + setState(454); string(); - setState(435); + setState(459); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(431); + setState(455); match(COMMA); - setState(432); + setState(456); string(); } } - setState(437); + setState(461); _errHandler.sync(this); _la = _input.LA(1); } - setState(438); + setState(462); match(CLOSING_BRACKET); } break; @@ -3572,16 +3764,16 @@ public T accept(ParseTreeVisitor visitor) { public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_parameter); + enterRule(_localctx, 70, RULE_parameter); try { - setState(444); + setState(468); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(442); + setState(466); match(PARAM); } break; @@ -3589,7 +3781,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(443); + setState(467); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3638,16 +3830,16 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierOrParameterContext identifierOrParameter() throws RecognitionException { IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_identifierOrParameter); + enterRule(_localctx, 72, RULE_identifierOrParameter); try { - setState(448); + setState(472); _errHandler.sync(this); switch (_input.LA(1)) { case UNQUOTED_IDENTIFIER: case QUOTED_IDENTIFIER: enterOuterAlt(_localctx, 1); { - setState(446); + setState(470); identifier(); } break; @@ -3655,7 +3847,7 @@ public final IdentifierOrParameterContext identifierOrParameter() throws Recogni case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(447); + setState(471); parameter(); } break; @@ -3700,13 +3892,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_limitCommand); + enterRule(_localctx, 74, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(450); + setState(474); match(LIMIT); - setState(451); + setState(475); match(INTEGER_LITERAL); } } @@ -3756,32 +3948,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_sortCommand); + enterRule(_localctx, 76, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(453); + setState(477); match(SORT); - setState(454); + setState(478); orderExpression(); - setState(459); + setState(483); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,45,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(455); + setState(479); match(COMMA); - setState(456); + setState(480); orderExpression(); } } } - setState(461); + setState(485); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,45,_ctx); } } } @@ -3830,19 +4022,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_orderExpression); + enterRule(_localctx, 78, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(462); + setState(486); booleanExpression(0); - setState(464); + setState(488); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: { - setState(463); + setState(487); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -3856,14 +4048,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(468); + setState(492); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: { - setState(466); + setState(490); match(NULLS); - setState(467); + setState(491); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -3918,13 +4110,13 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_keepCommand); + enterRule(_localctx, 80, RULE_keepCommand); try { enterOuterAlt(_localctx, 1); { - setState(470); + setState(494); match(KEEP); - setState(471); + setState(495); qualifiedNamePatterns(); } } @@ -3967,13 +4159,13 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_dropCommand); + enterRule(_localctx, 82, RULE_dropCommand); try { enterOuterAlt(_localctx, 1); { - setState(473); + setState(497); match(DROP); - setState(474); + setState(498); qualifiedNamePatterns(); } } @@ -4023,32 +4215,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_renameCommand); + enterRule(_localctx, 84, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(476); + setState(500); match(RENAME); - setState(477); + setState(501); renameClause(); - setState(482); + setState(506); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,48,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(478); + setState(502); match(COMMA); - setState(479); + setState(503); renameClause(); } } } - setState(484); + setState(508); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,48,_ctx); } } } @@ -4096,15 +4288,15 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_renameClause); + enterRule(_localctx, 86, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(485); + setState(509); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(486); + setState(510); match(AS); - setState(487); + setState(511); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4153,22 +4345,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_dissectCommand); + enterRule(_localctx, 88, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(489); + setState(513); match(DISSECT); - setState(490); + setState(514); primaryExpression(0); - setState(491); + setState(515); string(); - setState(493); + setState(517); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { case 1: { - setState(492); + setState(516); commandOptions(); } break; @@ -4217,15 +4409,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_grokCommand); + enterRule(_localctx, 90, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(495); + setState(519); match(GROK); - setState(496); + setState(520); primaryExpression(0); - setState(497); + setState(521); string(); } } @@ -4268,13 +4460,13 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_mvExpandCommand); + enterRule(_localctx, 92, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(499); + setState(523); match(MV_EXPAND); - setState(500); + setState(524); qualifiedName(); } } @@ -4323,30 +4515,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_commandOptions); + enterRule(_localctx, 94, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(502); + setState(526); commandOption(); - setState(507); + setState(531); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,47,_ctx); + _alt = getInterpreter().adaptivePredict(_input,50,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(503); + setState(527); match(COMMA); - setState(504); + setState(528); commandOption(); } } } - setState(509); + setState(533); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,47,_ctx); + _alt = getInterpreter().adaptivePredict(_input,50,_ctx); } } } @@ -4392,15 +4584,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_commandOption); + enterRule(_localctx, 96, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(510); + setState(534); identifier(); - setState(511); + setState(535); match(ASSIGN); - setState(512); + setState(536); constant(); } } @@ -4441,12 +4633,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_booleanValue); + enterRule(_localctx, 98, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(514); + setState(538); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4499,22 +4691,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 94, RULE_numericValue); + enterRule(_localctx, 100, RULE_numericValue); try { - setState(518); + setState(542); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(516); + setState(540); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(517); + setState(541); integerValue(); } break; @@ -4558,17 +4750,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 96, RULE_decimalValue); + enterRule(_localctx, 102, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(545); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(520); + setState(544); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4581,7 +4773,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(523); + setState(547); match(DECIMAL_LITERAL); } } @@ -4623,17 +4815,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 98, RULE_integerValue); + enterRule(_localctx, 104, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(526); + setState(550); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(525); + setState(549); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4646,7 +4838,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(528); + setState(552); match(INTEGER_LITERAL); } } @@ -4686,11 +4878,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 100, RULE_string); + enterRule(_localctx, 106, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(530); + setState(554); match(QUOTED_STRING); } } @@ -4735,14 +4927,14 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 102, RULE_comparisonOperator); + enterRule(_localctx, 108, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(532); + setState(556); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 562949953421312000L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 281474976710656000L) != 0)) ) { _errHandler.recoverInline(this); } else { @@ -4791,13 +4983,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 104, RULE_explainCommand); + enterRule(_localctx, 110, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(534); + setState(558); match(EXPLAIN); - setState(535); + setState(559); subqueryExpression(); } } @@ -4841,15 +5033,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 106, RULE_subqueryExpression); + enterRule(_localctx, 112, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(537); + setState(561); match(OPENING_BRACKET); - setState(538); + setState(562); query(0); - setState(539); + setState(563); match(CLOSING_BRACKET); } } @@ -4901,14 +5093,14 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 108, RULE_showCommand); + enterRule(_localctx, 114, RULE_showCommand); try { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(541); + setState(565); match(SHOW); - setState(542); + setState(566); match(INFO); } } @@ -4966,53 +5158,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 110, RULE_enrichCommand); + enterRule(_localctx, 116, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(544); + setState(568); match(ENRICH); - setState(545); + setState(569); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(548); + setState(572); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { case 1: { - setState(546); + setState(570); match(ON); - setState(547); + setState(571); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(559); + setState(583); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(550); + setState(574); match(WITH); - setState(551); + setState(575); enrichWithClause(); - setState(556); + setState(580); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,52,_ctx); + _alt = getInterpreter().adaptivePredict(_input,55,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(552); + setState(576); match(COMMA); - setState(553); + setState(577); enrichWithClause(); } } } - setState(558); + setState(582); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,52,_ctx); + _alt = getInterpreter().adaptivePredict(_input,55,_ctx); } } break; @@ -5063,23 +5255,23 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 112, RULE_enrichWithClause); + enterRule(_localctx, 118, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(564); + setState(588); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { case 1: { - setState(561); + setState(585); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(562); + setState(586); match(ASSIGN); } break; } - setState(566); + setState(590); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5128,17 +5320,17 @@ public T accept(ParseTreeVisitor visitor) { public final LookupCommandContext lookupCommand() throws RecognitionException { LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState()); - enterRule(_localctx, 114, RULE_lookupCommand); + enterRule(_localctx, 120, RULE_lookupCommand); try { enterOuterAlt(_localctx, 1); { - setState(568); + setState(592); match(DEV_LOOKUP); - setState(569); + setState(593); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(570); + setState(594); match(ON); - setState(571); + setState(595); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5155,16 +5347,16 @@ public final LookupCommandContext lookupCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class InlinestatsCommandContext extends ParserRuleContext { - public FieldsContext stats; + public AggFieldsContext stats; public FieldsContext grouping; public TerminalNode DEV_INLINESTATS() { return getToken(EsqlBaseParser.DEV_INLINESTATS, 0); } - public List fields() { - return getRuleContexts(FieldsContext.class); - } - public FieldsContext fields(int i) { - return getRuleContext(FieldsContext.class,i); + public AggFieldsContext aggFields() { + return getRuleContext(AggFieldsContext.class,0); } public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); } + public FieldsContext fields() { + return getRuleContext(FieldsContext.class,0); + } @SuppressWarnings("this-escape") public InlinestatsCommandContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -5187,22 +5379,22 @@ public T accept(ParseTreeVisitor visitor) { public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException { InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState()); - enterRule(_localctx, 116, RULE_inlinestatsCommand); + enterRule(_localctx, 122, RULE_inlinestatsCommand); try { enterOuterAlt(_localctx, 1); { - setState(573); + setState(597); match(DEV_INLINESTATS); - setState(574); - ((InlinestatsCommandContext)_localctx).stats = fields(); - setState(577); + setState(598); + ((InlinestatsCommandContext)_localctx).stats = aggFields(); + setState(601); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { case 1: { - setState(575); + setState(599); match(BY); - setState(576); + setState(600); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5234,6 +5426,8 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex); case 10: return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex); + case 28: + return aggField_sempred((AggFieldContext)_localctx, predIndex); } return true; } @@ -5287,9 +5481,16 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in } return true; } + private boolean aggField_sempred(AggFieldContext _localctx, int predIndex) { + switch (predIndex) { + case 10: + return this.isDevVersion(); + } + return true; + } public static final String _serializedATN = - "\u0004\u0001x\u0244\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ + "\u0004\u0001x\u025c\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ @@ -5304,360 +5505,373 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+ "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+ - "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0005\u0001\u0080\b\u0001\n\u0001\f\u0001\u0083\t\u0001\u0001"+ - "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003"+ - "\u0002\u008b\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002"+ + "<\u0007<\u0002=\u0007=\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001"+ + "\u0086\b\u0001\n\u0001\f\u0001\u0089\t\u0001\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0091\b\u0002\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003"+ - "\u0003\u009d\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+ - "\u0005\u00a9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0005\u0005\u00b0\b\u0005\n\u0005\f\u0005\u00b3\t\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00ba\b\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00c0\b\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0005\u0005\u00c8\b\u0005\n\u0005\f\u0005\u00cb\t\u0005\u0001\u0006\u0001"+ - "\u0006\u0003\u0006\u00cf\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0001\u0006\u0003\u0006\u00d6\b\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0003\u0006\u00db\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ - "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00e6\b\b\u0001"+ - "\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ec\b\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0001\t\u0005\t\u00f4\b\t\n\t\f\t\u00f7\t\t\u0001\n\u0001\n"+ - "\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0101\b\n\u0001"+ - "\n\u0001\n\u0001\n\u0005\n\u0106\b\n\n\n\f\n\u0109\t\n\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0111"+ - "\b\u000b\n\u000b\f\u000b\u0114\t\u000b\u0003\u000b\u0116\b\u000b\u0001"+ - "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0005\u000e\u0122\b\u000e\n\u000e\f\u000e\u0125"+ - "\t\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0003"+ - "\u000f\u012c\b\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005"+ - "\u0010\u0132\b\u0010\n\u0010\f\u0010\u0135\t\u0010\u0001\u0010\u0003\u0010"+ - "\u0138\b\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0003\u0011\u013f\b\u0011\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ - "\u0001\u0014\u0001\u0014\u0003\u0014\u0147\b\u0014\u0001\u0015\u0001\u0015"+ - "\u0001\u0015\u0001\u0015\u0005\u0015\u014d\b\u0015\n\u0015\f\u0015\u0150"+ - "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u015a\b\u0017\n\u0017\f\u0017"+ - "\u015d\t\u0017\u0001\u0017\u0003\u0017\u0160\b\u0017\u0001\u0017\u0001"+ - "\u0017\u0003\u0017\u0164\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001"+ - "\u0019\u0001\u0019\u0003\u0019\u016b\b\u0019\u0001\u0019\u0001\u0019\u0003"+ - "\u0019\u016f\b\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u0174"+ - "\b\u001a\n\u001a\f\u001a\u0177\t\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+ - "\u0005\u001b\u017c\b\u001b\n\u001b\f\u001b\u017f\t\u001b\u0001\u001c\u0001"+ - "\u001c\u0001\u001c\u0005\u001c\u0184\b\u001c\n\u001c\f\u001c\u0187\t\u001c"+ - "\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0003\u001e\u018d\b\u001e"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+ - "\u0001\u001f\u0005\u001f\u019c\b\u001f\n\u001f\f\u001f\u019f\t\u001f\u0001"+ - "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005"+ - "\u001f\u01a7\b\u001f\n\u001f\f\u001f\u01aa\t\u001f\u0001\u001f\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01b2\b\u001f"+ - "\n\u001f\f\u001f\u01b5\t\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01b9"+ - "\b\u001f\u0001 \u0001 \u0003 \u01bd\b \u0001!\u0001!\u0003!\u01c1\b!\u0001"+ - "\"\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0001#\u0005#\u01ca\b#\n#\f#\u01cd"+ - "\t#\u0001$\u0001$\u0003$\u01d1\b$\u0001$\u0001$\u0003$\u01d5\b$\u0001"+ - "%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0005"+ - "\'\u01e1\b\'\n\'\f\'\u01e4\t\'\u0001(\u0001(\u0001(\u0001(\u0001)\u0001"+ - ")\u0001)\u0001)\u0003)\u01ee\b)\u0001*\u0001*\u0001*\u0001*\u0001+\u0001"+ - "+\u0001+\u0001,\u0001,\u0001,\u0005,\u01fa\b,\n,\f,\u01fd\t,\u0001-\u0001"+ - "-\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0003/\u0207\b/\u00010\u0003"+ - "0\u020a\b0\u00010\u00010\u00011\u00031\u020f\b1\u00011\u00011\u00012\u0001"+ - "2\u00013\u00013\u00014\u00014\u00014\u00015\u00015\u00015\u00015\u0001"+ - "6\u00016\u00016\u00017\u00017\u00017\u00017\u00037\u0225\b7\u00017\u0001"+ - "7\u00017\u00017\u00057\u022b\b7\n7\f7\u022e\t7\u00037\u0230\b7\u00018"+ - "\u00018\u00018\u00038\u0235\b8\u00018\u00018\u00019\u00019\u00019\u0001"+ - "9\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u0242\b:\u0001:\u0000\u0004"+ - "\u0002\n\u0012\u0014;\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012"+ - "\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\"+ - "^`bdfhjlnprt\u0000\b\u0001\u0000;<\u0001\u0000=?\u0002\u0000\u001a\u001a"+ - "LL\u0001\u0000CD\u0002\u0000\u001f\u001f##\u0002\u0000&&))\u0002\u0000"+ - "%%33\u0002\u0000446:\u025e\u0000v\u0001\u0000\u0000\u0000\u0002y\u0001"+ - "\u0000\u0000\u0000\u0004\u008a\u0001\u0000\u0000\u0000\u0006\u009c\u0001"+ - "\u0000\u0000\u0000\b\u009e\u0001\u0000\u0000\u0000\n\u00bf\u0001\u0000"+ - "\u0000\u0000\f\u00da\u0001\u0000\u0000\u0000\u000e\u00dc\u0001\u0000\u0000"+ - "\u0000\u0010\u00e5\u0001\u0000\u0000\u0000\u0012\u00eb\u0001\u0000\u0000"+ - "\u0000\u0014\u0100\u0001\u0000\u0000\u0000\u0016\u010a\u0001\u0000\u0000"+ - "\u0000\u0018\u0119\u0001\u0000\u0000\u0000\u001a\u011b\u0001\u0000\u0000"+ - "\u0000\u001c\u011e\u0001\u0000\u0000\u0000\u001e\u012b\u0001\u0000\u0000"+ - "\u0000 \u012d\u0001\u0000\u0000\u0000\"\u013e\u0001\u0000\u0000\u0000"+ - "$\u0140\u0001\u0000\u0000\u0000&\u0142\u0001\u0000\u0000\u0000(\u0146"+ - "\u0001\u0000\u0000\u0000*\u0148\u0001\u0000\u0000\u0000,\u0151\u0001\u0000"+ - "\u0000\u0000.\u0155\u0001\u0000\u0000\u00000\u0165\u0001\u0000\u0000\u0000"+ - "2\u0168\u0001\u0000\u0000\u00004\u0170\u0001\u0000\u0000\u00006\u0178"+ - "\u0001\u0000\u0000\u00008\u0180\u0001\u0000\u0000\u0000:\u0188\u0001\u0000"+ - "\u0000\u0000<\u018c\u0001\u0000\u0000\u0000>\u01b8\u0001\u0000\u0000\u0000"+ - "@\u01bc\u0001\u0000\u0000\u0000B\u01c0\u0001\u0000\u0000\u0000D\u01c2"+ - "\u0001\u0000\u0000\u0000F\u01c5\u0001\u0000\u0000\u0000H\u01ce\u0001\u0000"+ - "\u0000\u0000J\u01d6\u0001\u0000\u0000\u0000L\u01d9\u0001\u0000\u0000\u0000"+ - "N\u01dc\u0001\u0000\u0000\u0000P\u01e5\u0001\u0000\u0000\u0000R\u01e9"+ - "\u0001\u0000\u0000\u0000T\u01ef\u0001\u0000\u0000\u0000V\u01f3\u0001\u0000"+ - "\u0000\u0000X\u01f6\u0001\u0000\u0000\u0000Z\u01fe\u0001\u0000\u0000\u0000"+ - "\\\u0202\u0001\u0000\u0000\u0000^\u0206\u0001\u0000\u0000\u0000`\u0209"+ - "\u0001\u0000\u0000\u0000b\u020e\u0001\u0000\u0000\u0000d\u0212\u0001\u0000"+ - "\u0000\u0000f\u0214\u0001\u0000\u0000\u0000h\u0216\u0001\u0000\u0000\u0000"+ - "j\u0219\u0001\u0000\u0000\u0000l\u021d\u0001\u0000\u0000\u0000n\u0220"+ - "\u0001\u0000\u0000\u0000p\u0234\u0001\u0000\u0000\u0000r\u0238\u0001\u0000"+ - "\u0000\u0000t\u023d\u0001\u0000\u0000\u0000vw\u0003\u0002\u0001\u0000"+ - "wx\u0005\u0000\u0000\u0001x\u0001\u0001\u0000\u0000\u0000yz\u0006\u0001"+ - "\uffff\uffff\u0000z{\u0003\u0004\u0002\u0000{\u0081\u0001\u0000\u0000"+ - "\u0000|}\n\u0001\u0000\u0000}~\u0005\u0019\u0000\u0000~\u0080\u0003\u0006"+ - "\u0003\u0000\u007f|\u0001\u0000\u0000\u0000\u0080\u0083\u0001\u0000\u0000"+ - "\u0000\u0081\u007f\u0001\u0000\u0000\u0000\u0081\u0082\u0001\u0000\u0000"+ - "\u0000\u0082\u0003\u0001\u0000\u0000\u0000\u0083\u0081\u0001\u0000\u0000"+ - "\u0000\u0084\u008b\u0003h4\u0000\u0085\u008b\u0003 \u0010\u0000\u0086"+ - "\u008b\u0003\u001a\r\u0000\u0087\u008b\u0003l6\u0000\u0088\u0089\u0004"+ - "\u0002\u0001\u0000\u0089\u008b\u0003.\u0017\u0000\u008a\u0084\u0001\u0000"+ - "\u0000\u0000\u008a\u0085\u0001\u0000\u0000\u0000\u008a\u0086\u0001\u0000"+ - "\u0000\u0000\u008a\u0087\u0001\u0000\u0000\u0000\u008a\u0088\u0001\u0000"+ - "\u0000\u0000\u008b\u0005\u0001\u0000\u0000\u0000\u008c\u009d\u00030\u0018"+ - "\u0000\u008d\u009d\u0003\b\u0004\u0000\u008e\u009d\u0003J%\u0000\u008f"+ - "\u009d\u0003D\"\u0000\u0090\u009d\u00032\u0019\u0000\u0091\u009d\u0003"+ - "F#\u0000\u0092\u009d\u0003L&\u0000\u0093\u009d\u0003N\'\u0000\u0094\u009d"+ - "\u0003R)\u0000\u0095\u009d\u0003T*\u0000\u0096\u009d\u0003n7\u0000\u0097"+ - "\u009d\u0003V+\u0000\u0098\u0099\u0004\u0003\u0002\u0000\u0099\u009d\u0003"+ - "t:\u0000\u009a\u009b\u0004\u0003\u0003\u0000\u009b\u009d\u0003r9\u0000"+ - "\u009c\u008c\u0001\u0000\u0000\u0000\u009c\u008d\u0001\u0000\u0000\u0000"+ - "\u009c\u008e\u0001\u0000\u0000\u0000\u009c\u008f\u0001\u0000\u0000\u0000"+ - "\u009c\u0090\u0001\u0000\u0000\u0000\u009c\u0091\u0001\u0000\u0000\u0000"+ - "\u009c\u0092\u0001\u0000\u0000\u0000\u009c\u0093\u0001\u0000\u0000\u0000"+ - "\u009c\u0094\u0001\u0000\u0000\u0000\u009c\u0095\u0001\u0000\u0000\u0000"+ - "\u009c\u0096\u0001\u0000\u0000\u0000\u009c\u0097\u0001\u0000\u0000\u0000"+ - "\u009c\u0098\u0001\u0000\u0000\u0000\u009c\u009a\u0001\u0000\u0000\u0000"+ - "\u009d\u0007\u0001\u0000\u0000\u0000\u009e\u009f\u0005\u0010\u0000\u0000"+ - "\u009f\u00a0\u0003\n\u0005\u0000\u00a0\t\u0001\u0000\u0000\u0000\u00a1"+ - "\u00a2\u0006\u0005\uffff\uffff\u0000\u00a2\u00a3\u0005,\u0000\u0000\u00a3"+ - "\u00c0\u0003\n\u0005\b\u00a4\u00c0\u0003\u0010\b\u0000\u00a5\u00c0\u0003"+ - "\f\u0006\u0000\u00a6\u00a8\u0003\u0010\b\u0000\u00a7\u00a9\u0005,\u0000"+ - "\u0000\u00a8\u00a7\u0001\u0000\u0000\u0000\u00a8\u00a9\u0001\u0000\u0000"+ - "\u0000\u00a9\u00aa\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\'\u0000\u0000"+ - "\u00ab\u00ac\u0005+\u0000\u0000\u00ac\u00b1\u0003\u0010\b\u0000\u00ad"+ - "\u00ae\u0005\"\u0000\u0000\u00ae\u00b0\u0003\u0010\b\u0000\u00af\u00ad"+ - "\u0001\u0000\u0000\u0000\u00b0\u00b3\u0001\u0000\u0000\u0000\u00b1\u00af"+ - "\u0001\u0000\u0000\u0000\u00b1\u00b2\u0001\u0000\u0000\u0000\u00b2\u00b4"+ - "\u0001\u0000\u0000\u0000\u00b3\u00b1\u0001\u0000\u0000\u0000\u00b4\u00b5"+ - "\u00052\u0000\u0000\u00b5\u00c0\u0001\u0000\u0000\u0000\u00b6\u00b7\u0003"+ - "\u0010\b\u0000\u00b7\u00b9\u0005(\u0000\u0000\u00b8\u00ba\u0005,\u0000"+ - "\u0000\u00b9\u00b8\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000\u0000"+ - "\u0000\u00ba\u00bb\u0001\u0000\u0000\u0000\u00bb\u00bc\u0005-\u0000\u0000"+ - "\u00bc\u00c0\u0001\u0000\u0000\u0000\u00bd\u00be\u0004\u0005\u0004\u0000"+ - "\u00be\u00c0\u0003\u000e\u0007\u0000\u00bf\u00a1\u0001\u0000\u0000\u0000"+ - "\u00bf\u00a4\u0001\u0000\u0000\u0000\u00bf\u00a5\u0001\u0000\u0000\u0000"+ - "\u00bf\u00a6\u0001\u0000\u0000\u0000\u00bf\u00b6\u0001\u0000\u0000\u0000"+ - "\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c9\u0001\u0000\u0000\u0000"+ - "\u00c1\u00c2\n\u0005\u0000\u0000\u00c2\u00c3\u0005\u001e\u0000\u0000\u00c3"+ - "\u00c8\u0003\n\u0005\u0006\u00c4\u00c5\n\u0004\u0000\u0000\u00c5\u00c6"+ - "\u0005/\u0000\u0000\u00c6\u00c8\u0003\n\u0005\u0005\u00c7\u00c1\u0001"+ - "\u0000\u0000\u0000\u00c7\u00c4\u0001\u0000\u0000\u0000\u00c8\u00cb\u0001"+ - "\u0000\u0000\u0000\u00c9\u00c7\u0001\u0000\u0000\u0000\u00c9\u00ca\u0001"+ - "\u0000\u0000\u0000\u00ca\u000b\u0001\u0000\u0000\u0000\u00cb\u00c9\u0001"+ - "\u0000\u0000\u0000\u00cc\u00ce\u0003\u0010\b\u0000\u00cd\u00cf\u0005,"+ - "\u0000\u0000\u00ce\u00cd\u0001\u0000\u0000\u0000\u00ce\u00cf\u0001\u0000"+ - "\u0000\u0000\u00cf\u00d0\u0001\u0000\u0000\u0000\u00d0\u00d1\u0005*\u0000"+ - "\u0000\u00d1\u00d2\u0003d2\u0000\u00d2\u00db\u0001\u0000\u0000\u0000\u00d3"+ - "\u00d5\u0003\u0010\b\u0000\u00d4\u00d6\u0005,\u0000\u0000\u00d5\u00d4"+ - "\u0001\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7"+ - "\u0001\u0000\u0000\u0000\u00d7\u00d8\u00051\u0000\u0000\u00d8\u00d9\u0003"+ - "d2\u0000\u00d9\u00db\u0001\u0000\u0000\u0000\u00da\u00cc\u0001\u0000\u0000"+ - "\u0000\u00da\u00d3\u0001\u0000\u0000\u0000\u00db\r\u0001\u0000\u0000\u0000"+ - "\u00dc\u00dd\u0003\u0010\b\u0000\u00dd\u00de\u0005\u0013\u0000\u0000\u00de"+ - "\u00df\u0003d2\u0000\u00df\u000f\u0001\u0000\u0000\u0000\u00e0\u00e6\u0003"+ - "\u0012\t\u0000\u00e1\u00e2\u0003\u0012\t\u0000\u00e2\u00e3\u0003f3\u0000"+ - "\u00e3\u00e4\u0003\u0012\t\u0000\u00e4\u00e6\u0001\u0000\u0000\u0000\u00e5"+ - "\u00e0\u0001\u0000\u0000\u0000\u00e5\u00e1\u0001\u0000\u0000\u0000\u00e6"+ - "\u0011\u0001\u0000\u0000\u0000\u00e7\u00e8\u0006\t\uffff\uffff\u0000\u00e8"+ - "\u00ec\u0003\u0014\n\u0000\u00e9\u00ea\u0007\u0000\u0000\u0000\u00ea\u00ec"+ - "\u0003\u0012\t\u0003\u00eb\u00e7\u0001\u0000\u0000\u0000\u00eb\u00e9\u0001"+ - "\u0000\u0000\u0000\u00ec\u00f5\u0001\u0000\u0000\u0000\u00ed\u00ee\n\u0002"+ - "\u0000\u0000\u00ee\u00ef\u0007\u0001\u0000\u0000\u00ef\u00f4\u0003\u0012"+ - "\t\u0003\u00f0\u00f1\n\u0001\u0000\u0000\u00f1\u00f2\u0007\u0000\u0000"+ - "\u0000\u00f2\u00f4\u0003\u0012\t\u0002\u00f3\u00ed\u0001\u0000\u0000\u0000"+ - "\u00f3\u00f0\u0001\u0000\u0000\u0000\u00f4\u00f7\u0001\u0000\u0000\u0000"+ - "\u00f5\u00f3\u0001\u0000\u0000\u0000\u00f5\u00f6\u0001\u0000\u0000\u0000"+ - "\u00f6\u0013\u0001\u0000\u0000\u0000\u00f7\u00f5\u0001\u0000\u0000\u0000"+ - "\u00f8\u00f9\u0006\n\uffff\uffff\u0000\u00f9\u0101\u0003>\u001f\u0000"+ - "\u00fa\u0101\u00034\u001a\u0000\u00fb\u0101\u0003\u0016\u000b\u0000\u00fc"+ - "\u00fd\u0005+\u0000\u0000\u00fd\u00fe\u0003\n\u0005\u0000\u00fe\u00ff"+ - "\u00052\u0000\u0000\u00ff\u0101\u0001\u0000\u0000\u0000\u0100\u00f8\u0001"+ - "\u0000\u0000\u0000\u0100\u00fa\u0001\u0000\u0000\u0000\u0100\u00fb\u0001"+ - "\u0000\u0000\u0000\u0100\u00fc\u0001\u0000\u0000\u0000\u0101\u0107\u0001"+ - "\u0000\u0000\u0000\u0102\u0103\n\u0001\u0000\u0000\u0103\u0104\u0005!"+ - "\u0000\u0000\u0104\u0106\u0003\u0018\f\u0000\u0105\u0102\u0001\u0000\u0000"+ - "\u0000\u0106\u0109\u0001\u0000\u0000\u0000\u0107\u0105\u0001\u0000\u0000"+ - "\u0000\u0107\u0108\u0001\u0000\u0000\u0000\u0108\u0015\u0001\u0000\u0000"+ - "\u0000\u0109\u0107\u0001\u0000\u0000\u0000\u010a\u010b\u0003B!\u0000\u010b"+ - "\u0115\u0005+\u0000\u0000\u010c\u0116\u0005=\u0000\u0000\u010d\u0112\u0003"+ - "\n\u0005\u0000\u010e\u010f\u0005\"\u0000\u0000\u010f\u0111\u0003\n\u0005"+ - "\u0000\u0110\u010e\u0001\u0000\u0000\u0000\u0111\u0114\u0001\u0000\u0000"+ - "\u0000\u0112\u0110\u0001\u0000\u0000\u0000\u0112\u0113\u0001\u0000\u0000"+ - "\u0000\u0113\u0116\u0001\u0000\u0000\u0000\u0114\u0112\u0001\u0000\u0000"+ - "\u0000\u0115\u010c\u0001\u0000\u0000\u0000\u0115\u010d\u0001\u0000\u0000"+ - "\u0000\u0115\u0116\u0001\u0000\u0000\u0000\u0116\u0117\u0001\u0000\u0000"+ - "\u0000\u0117\u0118\u00052\u0000\u0000\u0118\u0017\u0001\u0000\u0000\u0000"+ - "\u0119\u011a\u0003:\u001d\u0000\u011a\u0019\u0001\u0000\u0000\u0000\u011b"+ - "\u011c\u0005\f\u0000\u0000\u011c\u011d\u0003\u001c\u000e\u0000\u011d\u001b"+ - "\u0001\u0000\u0000\u0000\u011e\u0123\u0003\u001e\u000f\u0000\u011f\u0120"+ - "\u0005\"\u0000\u0000\u0120\u0122\u0003\u001e\u000f\u0000\u0121\u011f\u0001"+ - "\u0000\u0000\u0000\u0122\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001"+ - "\u0000\u0000\u0000\u0123\u0124\u0001\u0000\u0000\u0000\u0124\u001d\u0001"+ - "\u0000\u0000\u0000\u0125\u0123\u0001\u0000\u0000\u0000\u0126\u012c\u0003"+ - "\n\u0005\u0000\u0127\u0128\u00034\u001a\u0000\u0128\u0129\u0005 \u0000"+ - "\u0000\u0129\u012a\u0003\n\u0005\u0000\u012a\u012c\u0001\u0000\u0000\u0000"+ - "\u012b\u0126\u0001\u0000\u0000\u0000\u012b\u0127\u0001\u0000\u0000\u0000"+ - "\u012c\u001f\u0001\u0000\u0000\u0000\u012d\u012e\u0005\u0006\u0000\u0000"+ - "\u012e\u0133\u0003\"\u0011\u0000\u012f\u0130\u0005\"\u0000\u0000\u0130"+ - "\u0132\u0003\"\u0011\u0000\u0131\u012f\u0001\u0000\u0000\u0000\u0132\u0135"+ - "\u0001\u0000\u0000\u0000\u0133\u0131\u0001\u0000\u0000\u0000\u0133\u0134"+ - "\u0001\u0000\u0000\u0000\u0134\u0137\u0001\u0000\u0000\u0000\u0135\u0133"+ - "\u0001\u0000\u0000\u0000\u0136\u0138\u0003(\u0014\u0000\u0137\u0136\u0001"+ - "\u0000\u0000\u0000\u0137\u0138\u0001\u0000\u0000\u0000\u0138!\u0001\u0000"+ - "\u0000\u0000\u0139\u013a\u0003$\u0012\u0000\u013a\u013b\u0005h\u0000\u0000"+ - "\u013b\u013c\u0003&\u0013\u0000\u013c\u013f\u0001\u0000\u0000\u0000\u013d"+ - "\u013f\u0003&\u0013\u0000\u013e\u0139\u0001\u0000\u0000\u0000\u013e\u013d"+ - "\u0001\u0000\u0000\u0000\u013f#\u0001\u0000\u0000\u0000\u0140\u0141\u0005"+ - "L\u0000\u0000\u0141%\u0001\u0000\u0000\u0000\u0142\u0143\u0007\u0002\u0000"+ - "\u0000\u0143\'\u0001\u0000\u0000\u0000\u0144\u0147\u0003*\u0015\u0000"+ - "\u0145\u0147\u0003,\u0016\u0000\u0146\u0144\u0001\u0000\u0000\u0000\u0146"+ - "\u0145\u0001\u0000\u0000\u0000\u0147)\u0001\u0000\u0000\u0000\u0148\u0149"+ - "\u0005K\u0000\u0000\u0149\u014e\u0005L\u0000\u0000\u014a\u014b\u0005\""+ - "\u0000\u0000\u014b\u014d\u0005L\u0000\u0000\u014c\u014a\u0001\u0000\u0000"+ - "\u0000\u014d\u0150\u0001\u0000\u0000\u0000\u014e\u014c\u0001\u0000\u0000"+ - "\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f+\u0001\u0000\u0000\u0000"+ - "\u0150\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0005A\u0000\u0000\u0152"+ - "\u0153\u0003*\u0015\u0000\u0153\u0154\u0005B\u0000\u0000\u0154-\u0001"+ - "\u0000\u0000\u0000\u0155\u0156\u0005\u0014\u0000\u0000\u0156\u015b\u0003"+ - "\"\u0011\u0000\u0157\u0158\u0005\"\u0000\u0000\u0158\u015a\u0003\"\u0011"+ - "\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u015a\u015d\u0001\u0000\u0000"+ - "\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000\u0000"+ - "\u0000\u015c\u015f\u0001\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000"+ - "\u0000\u015e\u0160\u0003\u001c\u000e\u0000\u015f\u015e\u0001\u0000\u0000"+ - "\u0000\u015f\u0160\u0001\u0000\u0000\u0000\u0160\u0163\u0001\u0000\u0000"+ - "\u0000\u0161\u0162\u0005\u001d\u0000\u0000\u0162\u0164\u0003\u001c\u000e"+ - "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+ - "\u0000\u0164/\u0001\u0000\u0000\u0000\u0165\u0166\u0005\u0004\u0000\u0000"+ - "\u0166\u0167\u0003\u001c\u000e\u0000\u01671\u0001\u0000\u0000\u0000\u0168"+ - "\u016a\u0005\u000f\u0000\u0000\u0169\u016b\u0003\u001c\u000e\u0000\u016a"+ - "\u0169\u0001\u0000\u0000\u0000\u016a\u016b\u0001\u0000\u0000\u0000\u016b"+ - "\u016e\u0001\u0000\u0000\u0000\u016c\u016d\u0005\u001d\u0000\u0000\u016d"+ - "\u016f\u0003\u001c\u000e\u0000\u016e\u016c\u0001\u0000\u0000\u0000\u016e"+ - "\u016f\u0001\u0000\u0000\u0000\u016f3\u0001\u0000\u0000\u0000\u0170\u0175"+ - "\u0003B!\u0000\u0171\u0172\u0005$\u0000\u0000\u0172\u0174\u0003B!\u0000"+ - "\u0173\u0171\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000"+ - "\u0175\u0173\u0001\u0000\u0000\u0000\u0175\u0176\u0001\u0000\u0000\u0000"+ - "\u01765\u0001\u0000\u0000\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178"+ - "\u017d\u0003<\u001e\u0000\u0179\u017a\u0005$\u0000\u0000\u017a\u017c\u0003"+ - "<\u001e\u0000\u017b\u0179\u0001\u0000\u0000\u0000\u017c\u017f\u0001\u0000"+ - "\u0000\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017d\u017e\u0001\u0000"+ - "\u0000\u0000\u017e7\u0001\u0000\u0000\u0000\u017f\u017d\u0001\u0000\u0000"+ - "\u0000\u0180\u0185\u00036\u001b\u0000\u0181\u0182\u0005\"\u0000\u0000"+ - "\u0182\u0184\u00036\u001b\u0000\u0183\u0181\u0001\u0000\u0000\u0000\u0184"+ - "\u0187\u0001\u0000\u0000\u0000\u0185\u0183\u0001\u0000\u0000\u0000\u0185"+ - "\u0186\u0001\u0000\u0000\u0000\u01869\u0001\u0000\u0000\u0000\u0187\u0185"+ - "\u0001\u0000\u0000\u0000\u0188\u0189\u0007\u0003\u0000\u0000\u0189;\u0001"+ - "\u0000\u0000\u0000\u018a\u018d\u0005P\u0000\u0000\u018b\u018d\u0003@ "+ - "\u0000\u018c\u018a\u0001\u0000\u0000\u0000\u018c\u018b\u0001\u0000\u0000"+ - "\u0000\u018d=\u0001\u0000\u0000\u0000\u018e\u01b9\u0005-\u0000\u0000\u018f"+ - "\u0190\u0003b1\u0000\u0190\u0191\u0005C\u0000\u0000\u0191\u01b9\u0001"+ - "\u0000\u0000\u0000\u0192\u01b9\u0003`0\u0000\u0193\u01b9\u0003b1\u0000"+ - "\u0194\u01b9\u0003\\.\u0000\u0195\u01b9\u0003@ \u0000\u0196\u01b9\u0003"+ - "d2\u0000\u0197\u0198\u0005A\u0000\u0000\u0198\u019d\u0003^/\u0000\u0199"+ - "\u019a\u0005\"\u0000\u0000\u019a\u019c\u0003^/\u0000\u019b\u0199\u0001"+ - "\u0000\u0000\u0000\u019c\u019f\u0001\u0000\u0000\u0000\u019d\u019b\u0001"+ - "\u0000\u0000\u0000\u019d\u019e\u0001\u0000\u0000\u0000\u019e\u01a0\u0001"+ - "\u0000\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u01a0\u01a1\u0005"+ - "B\u0000\u0000\u01a1\u01b9\u0001\u0000\u0000\u0000\u01a2\u01a3\u0005A\u0000"+ - "\u0000\u01a3\u01a8\u0003\\.\u0000\u01a4\u01a5\u0005\"\u0000\u0000\u01a5"+ - "\u01a7\u0003\\.\u0000\u01a6\u01a4\u0001\u0000\u0000\u0000\u01a7\u01aa"+ - "\u0001\u0000\u0000\u0000\u01a8\u01a6\u0001\u0000\u0000\u0000\u01a8\u01a9"+ - "\u0001\u0000\u0000\u0000\u01a9\u01ab\u0001\u0000\u0000\u0000\u01aa\u01a8"+ - "\u0001\u0000\u0000\u0000\u01ab\u01ac\u0005B\u0000\u0000\u01ac\u01b9\u0001"+ - "\u0000\u0000\u0000\u01ad\u01ae\u0005A\u0000\u0000\u01ae\u01b3\u0003d2"+ - "\u0000\u01af\u01b0\u0005\"\u0000\u0000\u01b0\u01b2\u0003d2\u0000\u01b1"+ - "\u01af\u0001\u0000\u0000\u0000\u01b2\u01b5\u0001\u0000\u0000\u0000\u01b3"+ - "\u01b1\u0001\u0000\u0000\u0000\u01b3\u01b4\u0001\u0000\u0000\u0000\u01b4"+ - "\u01b6\u0001\u0000\u0000\u0000\u01b5\u01b3\u0001\u0000\u0000\u0000\u01b6"+ - "\u01b7\u0005B\u0000\u0000\u01b7\u01b9\u0001\u0000\u0000\u0000\u01b8\u018e"+ - "\u0001\u0000\u0000\u0000\u01b8\u018f\u0001\u0000\u0000\u0000\u01b8\u0192"+ - "\u0001\u0000\u0000\u0000\u01b8\u0193\u0001\u0000\u0000\u0000\u01b8\u0194"+ - "\u0001\u0000\u0000\u0000\u01b8\u0195\u0001\u0000\u0000\u0000\u01b8\u0196"+ - "\u0001\u0000\u0000\u0000\u01b8\u0197\u0001\u0000\u0000\u0000\u01b8\u01a2"+ - "\u0001\u0000\u0000\u0000\u01b8\u01ad\u0001\u0000\u0000\u0000\u01b9?\u0001"+ - "\u0000\u0000\u0000\u01ba\u01bd\u00050\u0000\u0000\u01bb\u01bd\u0005@\u0000"+ - "\u0000\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bc\u01bb\u0001\u0000\u0000"+ - "\u0000\u01bdA\u0001\u0000\u0000\u0000\u01be\u01c1\u0003:\u001d\u0000\u01bf"+ - "\u01c1\u0003@ \u0000\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01bf\u0001"+ - "\u0000\u0000\u0000\u01c1C\u0001\u0000\u0000\u0000\u01c2\u01c3\u0005\t"+ - "\u0000\u0000\u01c3\u01c4\u0005\u001b\u0000\u0000\u01c4E\u0001\u0000\u0000"+ - "\u0000\u01c5\u01c6\u0005\u000e\u0000\u0000\u01c6\u01cb\u0003H$\u0000\u01c7"+ - "\u01c8\u0005\"\u0000\u0000\u01c8\u01ca\u0003H$\u0000\u01c9\u01c7\u0001"+ - "\u0000\u0000\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001"+ - "\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01ccG\u0001\u0000"+ - "\u0000\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01d0\u0003\n\u0005"+ - "\u0000\u01cf\u01d1\u0007\u0004\u0000\u0000\u01d0\u01cf\u0001\u0000\u0000"+ - "\u0000\u01d0\u01d1\u0001\u0000\u0000\u0000\u01d1\u01d4\u0001\u0000\u0000"+ - "\u0000\u01d2\u01d3\u0005.\u0000\u0000\u01d3\u01d5\u0007\u0005\u0000\u0000"+ - "\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d4\u01d5\u0001\u0000\u0000\u0000"+ - "\u01d5I\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005\b\u0000\u0000\u01d7"+ - "\u01d8\u00038\u001c\u0000\u01d8K\u0001\u0000\u0000\u0000\u01d9\u01da\u0005"+ - "\u0002\u0000\u0000\u01da\u01db\u00038\u001c\u0000\u01dbM\u0001\u0000\u0000"+ - "\u0000\u01dc\u01dd\u0005\u000b\u0000\u0000\u01dd\u01e2\u0003P(\u0000\u01de"+ - "\u01df\u0005\"\u0000\u0000\u01df\u01e1\u0003P(\u0000\u01e0\u01de\u0001"+ - "\u0000\u0000\u0000\u01e1\u01e4\u0001\u0000\u0000\u0000\u01e2\u01e0\u0001"+ - "\u0000\u0000\u0000\u01e2\u01e3\u0001\u0000\u0000\u0000\u01e3O\u0001\u0000"+ - "\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5\u01e6\u00036\u001b"+ - "\u0000\u01e6\u01e7\u0005T\u0000\u0000\u01e7\u01e8\u00036\u001b\u0000\u01e8"+ - "Q\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005\u0001\u0000\u0000\u01ea\u01eb"+ - "\u0003\u0014\n\u0000\u01eb\u01ed\u0003d2\u0000\u01ec\u01ee\u0003X,\u0000"+ - "\u01ed\u01ec\u0001\u0000\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000\u0000"+ - "\u01eeS\u0001\u0000\u0000\u0000\u01ef\u01f0\u0005\u0007\u0000\u0000\u01f0"+ - "\u01f1\u0003\u0014\n\u0000\u01f1\u01f2\u0003d2\u0000\u01f2U\u0001\u0000"+ - "\u0000\u0000\u01f3\u01f4\u0005\n\u0000\u0000\u01f4\u01f5\u00034\u001a"+ - "\u0000\u01f5W\u0001\u0000\u0000\u0000\u01f6\u01fb\u0003Z-\u0000\u01f7"+ - "\u01f8\u0005\"\u0000\u0000\u01f8\u01fa\u0003Z-\u0000\u01f9\u01f7\u0001"+ - "\u0000\u0000\u0000\u01fa\u01fd\u0001\u0000\u0000\u0000\u01fb\u01f9\u0001"+ - "\u0000\u0000\u0000\u01fb\u01fc\u0001\u0000\u0000\u0000\u01fcY\u0001\u0000"+ - "\u0000\u0000\u01fd\u01fb\u0001\u0000\u0000\u0000\u01fe\u01ff\u0003:\u001d"+ - "\u0000\u01ff\u0200\u0005 \u0000\u0000\u0200\u0201\u0003>\u001f\u0000\u0201"+ - "[\u0001\u0000\u0000\u0000\u0202\u0203\u0007\u0006\u0000\u0000\u0203]\u0001"+ - "\u0000\u0000\u0000\u0204\u0207\u0003`0\u0000\u0205\u0207\u0003b1\u0000"+ - "\u0206\u0204\u0001\u0000\u0000\u0000\u0206\u0205\u0001\u0000\u0000\u0000"+ - "\u0207_\u0001\u0000\u0000\u0000\u0208\u020a\u0007\u0000\u0000\u0000\u0209"+ - "\u0208\u0001\u0000\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000\u020a"+ - "\u020b\u0001\u0000\u0000\u0000\u020b\u020c\u0005\u001c\u0000\u0000\u020c"+ - "a\u0001\u0000\u0000\u0000\u020d\u020f\u0007\u0000\u0000\u0000\u020e\u020d"+ - "\u0001\u0000\u0000\u0000\u020e\u020f\u0001\u0000\u0000\u0000\u020f\u0210"+ - "\u0001\u0000\u0000\u0000\u0210\u0211\u0005\u001b\u0000\u0000\u0211c\u0001"+ - "\u0000\u0000\u0000\u0212\u0213\u0005\u001a\u0000\u0000\u0213e\u0001\u0000"+ - "\u0000\u0000\u0214\u0215\u0007\u0007\u0000\u0000\u0215g\u0001\u0000\u0000"+ - "\u0000\u0216\u0217\u0005\u0005\u0000\u0000\u0217\u0218\u0003j5\u0000\u0218"+ - "i\u0001\u0000\u0000\u0000\u0219\u021a\u0005A\u0000\u0000\u021a\u021b\u0003"+ - "\u0002\u0001\u0000\u021b\u021c\u0005B\u0000\u0000\u021ck\u0001\u0000\u0000"+ - "\u0000\u021d\u021e\u0005\r\u0000\u0000\u021e\u021f\u0005d\u0000\u0000"+ - "\u021fm\u0001\u0000\u0000\u0000\u0220\u0221\u0005\u0003\u0000\u0000\u0221"+ - "\u0224\u0005Z\u0000\u0000\u0222\u0223\u0005X\u0000\u0000\u0223\u0225\u0003"+ - "6\u001b\u0000\u0224\u0222\u0001\u0000\u0000\u0000\u0224\u0225\u0001\u0000"+ - "\u0000\u0000\u0225\u022f\u0001\u0000\u0000\u0000\u0226\u0227\u0005Y\u0000"+ - "\u0000\u0227\u022c\u0003p8\u0000\u0228\u0229\u0005\"\u0000\u0000\u0229"+ - "\u022b\u0003p8\u0000\u022a\u0228\u0001\u0000\u0000\u0000\u022b\u022e\u0001"+ - "\u0000\u0000\u0000\u022c\u022a\u0001\u0000\u0000\u0000\u022c\u022d\u0001"+ - "\u0000\u0000\u0000\u022d\u0230\u0001\u0000\u0000\u0000\u022e\u022c\u0001"+ - "\u0000\u0000\u0000\u022f\u0226\u0001\u0000\u0000\u0000\u022f\u0230\u0001"+ - "\u0000\u0000\u0000\u0230o\u0001\u0000\u0000\u0000\u0231\u0232\u00036\u001b"+ - "\u0000\u0232\u0233\u0005 \u0000\u0000\u0233\u0235\u0001\u0000\u0000\u0000"+ - "\u0234\u0231\u0001\u0000\u0000\u0000\u0234\u0235\u0001\u0000\u0000\u0000"+ - "\u0235\u0236\u0001\u0000\u0000\u0000\u0236\u0237\u00036\u001b\u0000\u0237"+ - "q\u0001\u0000\u0000\u0000\u0238\u0239\u0005\u0012\u0000\u0000\u0239\u023a"+ - "\u0003\"\u0011\u0000\u023a\u023b\u0005X\u0000\u0000\u023b\u023c\u0003"+ - "8\u001c\u0000\u023cs\u0001\u0000\u0000\u0000\u023d\u023e\u0005\u0011\u0000"+ - "\u0000\u023e\u0241\u0003\u001c\u000e\u0000\u023f\u0240\u0005\u001d\u0000"+ - "\u0000\u0240\u0242\u0003\u001c\u000e\u0000\u0241\u023f\u0001\u0000\u0000"+ - "\u0000\u0241\u0242\u0001\u0000\u0000\u0000\u0242u\u0001\u0000\u0000\u0000"+ - "8\u0081\u008a\u009c\u00a8\u00b1\u00b9\u00bf\u00c7\u00c9\u00ce\u00d5\u00da"+ - "\u00e5\u00eb\u00f3\u00f5\u0100\u0107\u0112\u0115\u0123\u012b\u0133\u0137"+ - "\u013e\u0146\u014e\u015b\u015f\u0163\u016a\u016e\u0175\u017d\u0185\u018c"+ - "\u019d\u01a8\u01b3\u01b8\u01bc\u01c0\u01cb\u01d0\u01d4\u01e2\u01ed\u01fb"+ - "\u0206\u0209\u020e\u0224\u022c\u022f\u0234\u0241"; + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u00a3\b\u0003\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00af\b\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00b6"+ + "\b\u0005\n\u0005\f\u0005\u00b9\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0003\u0005\u00c0\b\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0003\u0005\u00c6\b\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00ce\b\u0005"+ + "\n\u0005\f\u0005\u00d1\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00d5"+ + "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00dc\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e1"+ + "\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001"+ + "\b\u0001\b\u0001\b\u0001\b\u0003\b\u00ec\b\b\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0003\t\u00f2\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005"+ + "\t\u00fa\b\t\n\t\f\t\u00fd\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n"+ + "\u0001\n\u0001\n\u0001\n\u0003\n\u0107\b\n\u0001\n\u0001\n\u0001\n\u0005"+ + "\n\u010c\b\n\n\n\f\n\u010f\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ + "\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0117\b\u000b\n\u000b\f\u000b"+ + "\u011a\t\u000b\u0003\u000b\u011c\b\u000b\u0001\u000b\u0001\u000b\u0001"+ + "\f\u0001\f\u0003\f\u0122\b\f\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001"+ + "\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0005\u000f\u012c\b\u000f\n"+ + "\u000f\f\u000f\u012f\t\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0003"+ + "\u0010\u0134\b\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001"+ + "\u0011\u0001\u0011\u0005\u0011\u013c\b\u0011\n\u0011\f\u0011\u013f\t\u0011"+ + "\u0001\u0011\u0003\u0011\u0142\b\u0011\u0001\u0012\u0001\u0012\u0001\u0012"+ + "\u0003\u0012\u0147\b\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ + "\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015\u0003\u0015\u0151\b\u0015"+ + "\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0157\b\u0016"+ + "\n\u0016\f\u0016\u015a\t\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+ + "\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018\u0164"+ + "\b\u0018\n\u0018\f\u0018\u0167\t\u0018\u0001\u0018\u0003\u0018\u016a\b"+ + "\u0018\u0001\u0018\u0001\u0018\u0003\u0018\u016e\b\u0018\u0001\u0019\u0001"+ + "\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0003\u001a\u0175\b\u001a\u0001"+ + "\u001a\u0001\u001a\u0003\u001a\u0179\b\u001a\u0001\u001b\u0001\u001b\u0001"+ + "\u001b\u0005\u001b\u017e\b\u001b\n\u001b\f\u001b\u0181\t\u001b\u0001\u001c"+ + "\u0001\u001c\u0001\u001c\u0001\u001c\u0003\u001c\u0187\b\u001c\u0001\u001d"+ + "\u0001\u001d\u0001\u001d\u0005\u001d\u018c\b\u001d\n\u001d\f\u001d\u018f"+ + "\t\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0005\u001e\u0194\b\u001e"+ + "\n\u001e\f\u001e\u0197\t\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0005"+ + "\u001f\u019c\b\u001f\n\u001f\f\u001f\u019f\t\u001f\u0001 \u0001 \u0001"+ + "!\u0001!\u0003!\u01a5\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ + "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01b4"+ + "\b\"\n\"\f\"\u01b7\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\""+ + "\u0005\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ + "\"\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001\"\u0001\"\u0003\""+ + "\u01d1\b\"\u0001#\u0001#\u0003#\u01d5\b#\u0001$\u0001$\u0003$\u01d9\b"+ + "$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0005&\u01e2\b&\n&"+ + "\f&\u01e5\t&\u0001\'\u0001\'\u0003\'\u01e9\b\'\u0001\'\u0001\'\u0003\'"+ + "\u01ed\b\'\u0001(\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001"+ + "*\u0001*\u0005*\u01f9\b*\n*\f*\u01fc\t*\u0001+\u0001+\u0001+\u0001+\u0001"+ + ",\u0001,\u0001,\u0001,\u0003,\u0206\b,\u0001-\u0001-\u0001-\u0001-\u0001"+ + ".\u0001.\u0001.\u0001/\u0001/\u0001/\u0005/\u0212\b/\n/\f/\u0215\t/\u0001"+ + "0\u00010\u00010\u00010\u00011\u00011\u00012\u00012\u00032\u021f\b2\u0001"+ + "3\u00033\u0222\b3\u00013\u00013\u00014\u00034\u0227\b4\u00014\u00014\u0001"+ + "5\u00015\u00016\u00016\u00017\u00017\u00017\u00018\u00018\u00018\u0001"+ + "8\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u023d\b:\u0001"+ + ":\u0001:\u0001:\u0001:\u0005:\u0243\b:\n:\f:\u0246\t:\u0003:\u0248\b:"+ + "\u0001;\u0001;\u0001;\u0003;\u024d\b;\u0001;\u0001;\u0001<\u0001<\u0001"+ + "<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0003=\u025a\b=\u0001=\u0000"+ + "\u0004\u0002\n\u0012\u0014>\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`bdfhjlnprtvxz\u0000\b\u0001\u0000:;\u0001\u0000<>\u0002\u0000"+ + "\u0019\u0019LL\u0001\u0000CD\u0002\u0000\u001e\u001e\"\"\u0002\u0000%"+ + "%((\u0002\u0000$$22\u0002\u00003359\u0276\u0000|\u0001\u0000\u0000\u0000"+ + "\u0002\u007f\u0001\u0000\u0000\u0000\u0004\u0090\u0001\u0000\u0000\u0000"+ + "\u0006\u00a2\u0001\u0000\u0000\u0000\b\u00a4\u0001\u0000\u0000\u0000\n"+ + "\u00c5\u0001\u0000\u0000\u0000\f\u00e0\u0001\u0000\u0000\u0000\u000e\u00e2"+ + "\u0001\u0000\u0000\u0000\u0010\u00eb\u0001\u0000\u0000\u0000\u0012\u00f1"+ + "\u0001\u0000\u0000\u0000\u0014\u0106\u0001\u0000\u0000\u0000\u0016\u0110"+ + "\u0001\u0000\u0000\u0000\u0018\u0121\u0001\u0000\u0000\u0000\u001a\u0123"+ + "\u0001\u0000\u0000\u0000\u001c\u0125\u0001\u0000\u0000\u0000\u001e\u0128"+ + "\u0001\u0000\u0000\u0000 \u0133\u0001\u0000\u0000\u0000\"\u0137\u0001"+ + "\u0000\u0000\u0000$\u0146\u0001\u0000\u0000\u0000&\u014a\u0001\u0000\u0000"+ + "\u0000(\u014c\u0001\u0000\u0000\u0000*\u0150\u0001\u0000\u0000\u0000,"+ + "\u0152\u0001\u0000\u0000\u0000.\u015b\u0001\u0000\u0000\u00000\u015f\u0001"+ + "\u0000\u0000\u00002\u016f\u0001\u0000\u0000\u00004\u0172\u0001\u0000\u0000"+ + "\u00006\u017a\u0001\u0000\u0000\u00008\u0182\u0001\u0000\u0000\u0000:"+ + "\u0188\u0001\u0000\u0000\u0000<\u0190\u0001\u0000\u0000\u0000>\u0198\u0001"+ + "\u0000\u0000\u0000@\u01a0\u0001\u0000\u0000\u0000B\u01a4\u0001\u0000\u0000"+ + "\u0000D\u01d0\u0001\u0000\u0000\u0000F\u01d4\u0001\u0000\u0000\u0000H"+ + "\u01d8\u0001\u0000\u0000\u0000J\u01da\u0001\u0000\u0000\u0000L\u01dd\u0001"+ + "\u0000\u0000\u0000N\u01e6\u0001\u0000\u0000\u0000P\u01ee\u0001\u0000\u0000"+ + "\u0000R\u01f1\u0001\u0000\u0000\u0000T\u01f4\u0001\u0000\u0000\u0000V"+ + "\u01fd\u0001\u0000\u0000\u0000X\u0201\u0001\u0000\u0000\u0000Z\u0207\u0001"+ + "\u0000\u0000\u0000\\\u020b\u0001\u0000\u0000\u0000^\u020e\u0001\u0000"+ + "\u0000\u0000`\u0216\u0001\u0000\u0000\u0000b\u021a\u0001\u0000\u0000\u0000"+ + "d\u021e\u0001\u0000\u0000\u0000f\u0221\u0001\u0000\u0000\u0000h\u0226"+ + "\u0001\u0000\u0000\u0000j\u022a\u0001\u0000\u0000\u0000l\u022c\u0001\u0000"+ + "\u0000\u0000n\u022e\u0001\u0000\u0000\u0000p\u0231\u0001\u0000\u0000\u0000"+ + "r\u0235\u0001\u0000\u0000\u0000t\u0238\u0001\u0000\u0000\u0000v\u024c"+ + "\u0001\u0000\u0000\u0000x\u0250\u0001\u0000\u0000\u0000z\u0255\u0001\u0000"+ + "\u0000\u0000|}\u0003\u0002\u0001\u0000}~\u0005\u0000\u0000\u0001~\u0001"+ + "\u0001\u0000\u0000\u0000\u007f\u0080\u0006\u0001\uffff\uffff\u0000\u0080"+ + "\u0081\u0003\u0004\u0002\u0000\u0081\u0087\u0001\u0000\u0000\u0000\u0082"+ + "\u0083\n\u0001\u0000\u0000\u0083\u0084\u0005\u0018\u0000\u0000\u0084\u0086"+ + "\u0003\u0006\u0003\u0000\u0085\u0082\u0001\u0000\u0000\u0000\u0086\u0089"+ + "\u0001\u0000\u0000\u0000\u0087\u0085\u0001\u0000\u0000\u0000\u0087\u0088"+ + "\u0001\u0000\u0000\u0000\u0088\u0003\u0001\u0000\u0000\u0000\u0089\u0087"+ + "\u0001\u0000\u0000\u0000\u008a\u0091\u0003n7\u0000\u008b\u0091\u0003\""+ + "\u0011\u0000\u008c\u0091\u0003\u001c\u000e\u0000\u008d\u0091\u0003r9\u0000"+ + "\u008e\u008f\u0004\u0002\u0001\u0000\u008f\u0091\u00030\u0018\u0000\u0090"+ + "\u008a\u0001\u0000\u0000\u0000\u0090\u008b\u0001\u0000\u0000\u0000\u0090"+ + "\u008c\u0001\u0000\u0000\u0000\u0090\u008d\u0001\u0000\u0000\u0000\u0090"+ + "\u008e\u0001\u0000\u0000\u0000\u0091\u0005\u0001\u0000\u0000\u0000\u0092"+ + "\u00a3\u00032\u0019\u0000\u0093\u00a3\u0003\b\u0004\u0000\u0094\u00a3"+ + "\u0003P(\u0000\u0095\u00a3\u0003J%\u0000\u0096\u00a3\u00034\u001a\u0000"+ + "\u0097\u00a3\u0003L&\u0000\u0098\u00a3\u0003R)\u0000\u0099\u00a3\u0003"+ + "T*\u0000\u009a\u00a3\u0003X,\u0000\u009b\u00a3\u0003Z-\u0000\u009c\u00a3"+ + "\u0003t:\u0000\u009d\u00a3\u0003\\.\u0000\u009e\u009f\u0004\u0003\u0002"+ + "\u0000\u009f\u00a3\u0003z=\u0000\u00a0\u00a1\u0004\u0003\u0003\u0000\u00a1"+ + "\u00a3\u0003x<\u0000\u00a2\u0092\u0001\u0000\u0000\u0000\u00a2\u0093\u0001"+ + "\u0000\u0000\u0000\u00a2\u0094\u0001\u0000\u0000\u0000\u00a2\u0095\u0001"+ + "\u0000\u0000\u0000\u00a2\u0096\u0001\u0000\u0000\u0000\u00a2\u0097\u0001"+ + "\u0000\u0000\u0000\u00a2\u0098\u0001\u0000\u0000\u0000\u00a2\u0099\u0001"+ + "\u0000\u0000\u0000\u00a2\u009a\u0001\u0000\u0000\u0000\u00a2\u009b\u0001"+ + "\u0000\u0000\u0000\u00a2\u009c\u0001\u0000\u0000\u0000\u00a2\u009d\u0001"+ + "\u0000\u0000\u0000\u00a2\u009e\u0001\u0000\u0000\u0000\u00a2\u00a0\u0001"+ + "\u0000\u0000\u0000\u00a3\u0007\u0001\u0000\u0000\u0000\u00a4\u00a5\u0005"+ + "\u0010\u0000\u0000\u00a5\u00a6\u0003\n\u0005\u0000\u00a6\t\u0001\u0000"+ + "\u0000\u0000\u00a7\u00a8\u0006\u0005\uffff\uffff\u0000\u00a8\u00a9\u0005"+ + "+\u0000\u0000\u00a9\u00c6\u0003\n\u0005\b\u00aa\u00c6\u0003\u0010\b\u0000"+ + "\u00ab\u00c6\u0003\f\u0006\u0000\u00ac\u00ae\u0003\u0010\b\u0000\u00ad"+ + "\u00af\u0005+\u0000\u0000\u00ae\u00ad\u0001\u0000\u0000\u0000\u00ae\u00af"+ + "\u0001\u0000\u0000\u0000\u00af\u00b0\u0001\u0000\u0000\u0000\u00b0\u00b1"+ + "\u0005&\u0000\u0000\u00b1\u00b2\u0005*\u0000\u0000\u00b2\u00b7\u0003\u0010"+ + "\b\u0000\u00b3\u00b4\u0005!\u0000\u0000\u00b4\u00b6\u0003\u0010\b\u0000"+ + "\u00b5\u00b3\u0001\u0000\u0000\u0000\u00b6\u00b9\u0001\u0000\u0000\u0000"+ + "\u00b7\u00b5\u0001\u0000\u0000\u0000\u00b7\u00b8\u0001\u0000\u0000\u0000"+ + "\u00b8\u00ba\u0001\u0000\u0000\u0000\u00b9\u00b7\u0001\u0000\u0000\u0000"+ + "\u00ba\u00bb\u00051\u0000\u0000\u00bb\u00c6\u0001\u0000\u0000\u0000\u00bc"+ + "\u00bd\u0003\u0010\b\u0000\u00bd\u00bf\u0005\'\u0000\u0000\u00be\u00c0"+ + "\u0005+\u0000\u0000\u00bf\u00be\u0001\u0000\u0000\u0000\u00bf\u00c0\u0001"+ + "\u0000\u0000\u0000\u00c0\u00c1\u0001\u0000\u0000\u0000\u00c1\u00c2\u0005"+ + ",\u0000\u0000\u00c2\u00c6\u0001\u0000\u0000\u0000\u00c3\u00c4\u0004\u0005"+ + "\u0004\u0000\u00c4\u00c6\u0003\u000e\u0007\u0000\u00c5\u00a7\u0001\u0000"+ + "\u0000\u0000\u00c5\u00aa\u0001\u0000\u0000\u0000\u00c5\u00ab\u0001\u0000"+ + "\u0000\u0000\u00c5\u00ac\u0001\u0000\u0000\u0000\u00c5\u00bc\u0001\u0000"+ + "\u0000\u0000\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c6\u00cf\u0001\u0000"+ + "\u0000\u0000\u00c7\u00c8\n\u0005\u0000\u0000\u00c8\u00c9\u0005\u001d\u0000"+ + "\u0000\u00c9\u00ce\u0003\n\u0005\u0006\u00ca\u00cb\n\u0004\u0000\u0000"+ + "\u00cb\u00cc\u0005.\u0000\u0000\u00cc\u00ce\u0003\n\u0005\u0005\u00cd"+ + "\u00c7\u0001\u0000\u0000\u0000\u00cd\u00ca\u0001\u0000\u0000\u0000\u00ce"+ + "\u00d1\u0001\u0000\u0000\u0000\u00cf\u00cd\u0001\u0000\u0000\u0000\u00cf"+ + "\u00d0\u0001\u0000\u0000\u0000\u00d0\u000b\u0001\u0000\u0000\u0000\u00d1"+ + "\u00cf\u0001\u0000\u0000\u0000\u00d2\u00d4\u0003\u0010\b\u0000\u00d3\u00d5"+ + "\u0005+\u0000\u0000\u00d4\u00d3\u0001\u0000\u0000\u0000\u00d4\u00d5\u0001"+ + "\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7\u0005"+ + ")\u0000\u0000\u00d7\u00d8\u0003j5\u0000\u00d8\u00e1\u0001\u0000\u0000"+ + "\u0000\u00d9\u00db\u0003\u0010\b\u0000\u00da\u00dc\u0005+\u0000\u0000"+ + "\u00db\u00da\u0001\u0000\u0000\u0000\u00db\u00dc\u0001\u0000\u0000\u0000"+ + "\u00dc\u00dd\u0001\u0000\u0000\u0000\u00dd\u00de\u00050\u0000\u0000\u00de"+ + "\u00df\u0003j5\u0000\u00df\u00e1\u0001\u0000\u0000\u0000\u00e0\u00d2\u0001"+ + "\u0000\u0000\u0000\u00e0\u00d9\u0001\u0000\u0000\u0000\u00e1\r\u0001\u0000"+ + "\u0000\u0000\u00e2\u00e3\u0003\u0010\b\u0000\u00e3\u00e4\u0005?\u0000"+ + "\u0000\u00e4\u00e5\u0003j5\u0000\u00e5\u000f\u0001\u0000\u0000\u0000\u00e6"+ + "\u00ec\u0003\u0012\t\u0000\u00e7\u00e8\u0003\u0012\t\u0000\u00e8\u00e9"+ + "\u0003l6\u0000\u00e9\u00ea\u0003\u0012\t\u0000\u00ea\u00ec\u0001\u0000"+ + "\u0000\u0000\u00eb\u00e6\u0001\u0000\u0000\u0000\u00eb\u00e7\u0001\u0000"+ + "\u0000\u0000\u00ec\u0011\u0001\u0000\u0000\u0000\u00ed\u00ee\u0006\t\uffff"+ + "\uffff\u0000\u00ee\u00f2\u0003\u0014\n\u0000\u00ef\u00f0\u0007\u0000\u0000"+ + "\u0000\u00f0\u00f2\u0003\u0012\t\u0003\u00f1\u00ed\u0001\u0000\u0000\u0000"+ + "\u00f1\u00ef\u0001\u0000\u0000\u0000\u00f2\u00fb\u0001\u0000\u0000\u0000"+ + "\u00f3\u00f4\n\u0002\u0000\u0000\u00f4\u00f5\u0007\u0001\u0000\u0000\u00f5"+ + "\u00fa\u0003\u0012\t\u0003\u00f6\u00f7\n\u0001\u0000\u0000\u00f7\u00f8"+ + "\u0007\u0000\u0000\u0000\u00f8\u00fa\u0003\u0012\t\u0002\u00f9\u00f3\u0001"+ + "\u0000\u0000\u0000\u00f9\u00f6\u0001\u0000\u0000\u0000\u00fa\u00fd\u0001"+ + "\u0000\u0000\u0000\u00fb\u00f9\u0001\u0000\u0000\u0000\u00fb\u00fc\u0001"+ + "\u0000\u0000\u0000\u00fc\u0013\u0001\u0000\u0000\u0000\u00fd\u00fb\u0001"+ + "\u0000\u0000\u0000\u00fe\u00ff\u0006\n\uffff\uffff\u0000\u00ff\u0107\u0003"+ + "D\"\u0000\u0100\u0107\u0003:\u001d\u0000\u0101\u0107\u0003\u0016\u000b"+ + "\u0000\u0102\u0103\u0005*\u0000\u0000\u0103\u0104\u0003\n\u0005\u0000"+ + "\u0104\u0105\u00051\u0000\u0000\u0105\u0107\u0001\u0000\u0000\u0000\u0106"+ + "\u00fe\u0001\u0000\u0000\u0000\u0106\u0100\u0001\u0000\u0000\u0000\u0106"+ + "\u0101\u0001\u0000\u0000\u0000\u0106\u0102\u0001\u0000\u0000\u0000\u0107"+ + "\u010d\u0001\u0000\u0000\u0000\u0108\u0109\n\u0001\u0000\u0000\u0109\u010a"+ + "\u0005 \u0000\u0000\u010a\u010c\u0003\u001a\r\u0000\u010b\u0108\u0001"+ + "\u0000\u0000\u0000\u010c\u010f\u0001\u0000\u0000\u0000\u010d\u010b\u0001"+ + "\u0000\u0000\u0000\u010d\u010e\u0001\u0000\u0000\u0000\u010e\u0015\u0001"+ + "\u0000\u0000\u0000\u010f\u010d\u0001\u0000\u0000\u0000\u0110\u0111\u0003"+ + "\u0018\f\u0000\u0111\u011b\u0005*\u0000\u0000\u0112\u011c\u0005<\u0000"+ + "\u0000\u0113\u0118\u0003\n\u0005\u0000\u0114\u0115\u0005!\u0000\u0000"+ + "\u0115\u0117\u0003\n\u0005\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0117"+ + "\u011a\u0001\u0000\u0000\u0000\u0118\u0116\u0001\u0000\u0000\u0000\u0118"+ + "\u0119\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000\u011a"+ + "\u0118\u0001\u0000\u0000\u0000\u011b\u0112\u0001\u0000\u0000\u0000\u011b"+ + "\u0113\u0001\u0000\u0000\u0000\u011b\u011c\u0001\u0000\u0000\u0000\u011c"+ + "\u011d\u0001\u0000\u0000\u0000\u011d\u011e\u00051\u0000\u0000\u011e\u0017"+ + "\u0001\u0000\u0000\u0000\u011f\u0122\u0005?\u0000\u0000\u0120\u0122\u0003"+ + "H$\u0000\u0121\u011f\u0001\u0000\u0000\u0000\u0121\u0120\u0001\u0000\u0000"+ + "\u0000\u0122\u0019\u0001\u0000\u0000\u0000\u0123\u0124\u0003@ \u0000\u0124"+ + "\u001b\u0001\u0000\u0000\u0000\u0125\u0126\u0005\f\u0000\u0000\u0126\u0127"+ + "\u0003\u001e\u000f\u0000\u0127\u001d\u0001\u0000\u0000\u0000\u0128\u012d"+ + "\u0003 \u0010\u0000\u0129\u012a\u0005!\u0000\u0000\u012a\u012c\u0003 "+ + "\u0010\u0000\u012b\u0129\u0001\u0000\u0000\u0000\u012c\u012f\u0001\u0000"+ + "\u0000\u0000\u012d\u012b\u0001\u0000\u0000\u0000\u012d\u012e\u0001\u0000"+ + "\u0000\u0000\u012e\u001f\u0001\u0000\u0000\u0000\u012f\u012d\u0001\u0000"+ + "\u0000\u0000\u0130\u0131\u0003:\u001d\u0000\u0131\u0132\u0005\u001f\u0000"+ + "\u0000\u0132\u0134\u0001\u0000\u0000\u0000\u0133\u0130\u0001\u0000\u0000"+ + "\u0000\u0133\u0134\u0001\u0000\u0000\u0000\u0134\u0135\u0001\u0000\u0000"+ + "\u0000\u0135\u0136\u0003\n\u0005\u0000\u0136!\u0001\u0000\u0000\u0000"+ + "\u0137\u0138\u0005\u0006\u0000\u0000\u0138\u013d\u0003$\u0012\u0000\u0139"+ + "\u013a\u0005!\u0000\u0000\u013a\u013c\u0003$\u0012\u0000\u013b\u0139\u0001"+ + "\u0000\u0000\u0000\u013c\u013f\u0001\u0000\u0000\u0000\u013d\u013b\u0001"+ + "\u0000\u0000\u0000\u013d\u013e\u0001\u0000\u0000\u0000\u013e\u0141\u0001"+ + "\u0000\u0000\u0000\u013f\u013d\u0001\u0000\u0000\u0000\u0140\u0142\u0003"+ + "*\u0015\u0000\u0141\u0140\u0001\u0000\u0000\u0000\u0141\u0142\u0001\u0000"+ + "\u0000\u0000\u0142#\u0001\u0000\u0000\u0000\u0143\u0144\u0003&\u0013\u0000"+ + "\u0144\u0145\u0005h\u0000\u0000\u0145\u0147\u0001\u0000\u0000\u0000\u0146"+ + "\u0143\u0001\u0000\u0000\u0000\u0146\u0147\u0001\u0000\u0000\u0000\u0147"+ + "\u0148\u0001\u0000\u0000\u0000\u0148\u0149\u0003(\u0014\u0000\u0149%\u0001"+ + "\u0000\u0000\u0000\u014a\u014b\u0005L\u0000\u0000\u014b\'\u0001\u0000"+ + "\u0000\u0000\u014c\u014d\u0007\u0002\u0000\u0000\u014d)\u0001\u0000\u0000"+ + "\u0000\u014e\u0151\u0003,\u0016\u0000\u014f\u0151\u0003.\u0017\u0000\u0150"+ + "\u014e\u0001\u0000\u0000\u0000\u0150\u014f\u0001\u0000\u0000\u0000\u0151"+ + "+\u0001\u0000\u0000\u0000\u0152\u0153\u0005K\u0000\u0000\u0153\u0158\u0005"+ + "L\u0000\u0000\u0154\u0155\u0005!\u0000\u0000\u0155\u0157\u0005L\u0000"+ + "\u0000\u0156\u0154\u0001\u0000\u0000\u0000\u0157\u015a\u0001\u0000\u0000"+ + "\u0000\u0158\u0156\u0001\u0000\u0000\u0000\u0158\u0159\u0001\u0000\u0000"+ + "\u0000\u0159-\u0001\u0000\u0000\u0000\u015a\u0158\u0001\u0000\u0000\u0000"+ + "\u015b\u015c\u0005A\u0000\u0000\u015c\u015d\u0003,\u0016\u0000\u015d\u015e"+ + "\u0005B\u0000\u0000\u015e/\u0001\u0000\u0000\u0000\u015f\u0160\u0005\u0013"+ + "\u0000\u0000\u0160\u0165\u0003$\u0012\u0000\u0161\u0162\u0005!\u0000\u0000"+ + "\u0162\u0164\u0003$\u0012\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0164"+ + "\u0167\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000\u0165"+ + "\u0166\u0001\u0000\u0000\u0000\u0166\u0169\u0001\u0000\u0000\u0000\u0167"+ + "\u0165\u0001\u0000\u0000\u0000\u0168\u016a\u00036\u001b\u0000\u0169\u0168"+ + "\u0001\u0000\u0000\u0000\u0169\u016a\u0001\u0000\u0000\u0000\u016a\u016d"+ + "\u0001\u0000\u0000\u0000\u016b\u016c\u0005\u001c\u0000\u0000\u016c\u016e"+ + "\u0003\u001e\u000f\u0000\u016d\u016b\u0001\u0000\u0000\u0000\u016d\u016e"+ + "\u0001\u0000\u0000\u0000\u016e1\u0001\u0000\u0000\u0000\u016f\u0170\u0005"+ + "\u0004\u0000\u0000\u0170\u0171\u0003\u001e\u000f\u0000\u01713\u0001\u0000"+ + "\u0000\u0000\u0172\u0174\u0005\u000f\u0000\u0000\u0173\u0175\u00036\u001b"+ + "\u0000\u0174\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000"+ + "\u0000\u0175\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005\u001c\u0000"+ + "\u0000\u0177\u0179\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000"+ + "\u0000\u0178\u0179\u0001\u0000\u0000\u0000\u01795\u0001\u0000\u0000\u0000"+ + "\u017a\u017f\u00038\u001c\u0000\u017b\u017c\u0005!\u0000\u0000\u017c\u017e"+ + "\u00038\u001c\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017e\u0181\u0001"+ + "\u0000\u0000\u0000\u017f\u017d\u0001\u0000\u0000\u0000\u017f\u0180\u0001"+ + "\u0000\u0000\u0000\u01807\u0001\u0000\u0000\u0000\u0181\u017f\u0001\u0000"+ + "\u0000\u0000\u0182\u0183\u0003 \u0010\u0000\u0183\u0186\u0004\u001c\n"+ + "\u0000\u0184\u0185\u0005\u0010\u0000\u0000\u0185\u0187\u0003\n\u0005\u0000"+ + "\u0186\u0184\u0001\u0000\u0000\u0000\u0186\u0187\u0001\u0000\u0000\u0000"+ + "\u01879\u0001\u0000\u0000\u0000\u0188\u018d\u0003H$\u0000\u0189\u018a"+ + "\u0005#\u0000\u0000\u018a\u018c\u0003H$\u0000\u018b\u0189\u0001\u0000"+ + "\u0000\u0000\u018c\u018f\u0001\u0000\u0000\u0000\u018d\u018b\u0001\u0000"+ + "\u0000\u0000\u018d\u018e\u0001\u0000\u0000\u0000\u018e;\u0001\u0000\u0000"+ + "\u0000\u018f\u018d\u0001\u0000\u0000\u0000\u0190\u0195\u0003B!\u0000\u0191"+ + "\u0192\u0005#\u0000\u0000\u0192\u0194\u0003B!\u0000\u0193\u0191\u0001"+ + "\u0000\u0000\u0000\u0194\u0197\u0001\u0000\u0000\u0000\u0195\u0193\u0001"+ + "\u0000\u0000\u0000\u0195\u0196\u0001\u0000\u0000\u0000\u0196=\u0001\u0000"+ + "\u0000\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0198\u019d\u0003<\u001e"+ + "\u0000\u0199\u019a\u0005!\u0000\u0000\u019a\u019c\u0003<\u001e\u0000\u019b"+ + "\u0199\u0001\u0000\u0000\u0000\u019c\u019f\u0001\u0000\u0000\u0000\u019d"+ + "\u019b\u0001\u0000\u0000\u0000\u019d\u019e\u0001\u0000\u0000\u0000\u019e"+ + "?\u0001\u0000\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u01a0\u01a1"+ + "\u0007\u0003\u0000\u0000\u01a1A\u0001\u0000\u0000\u0000\u01a2\u01a5\u0005"+ + "P\u0000\u0000\u01a3\u01a5\u0003F#\u0000\u01a4\u01a2\u0001\u0000\u0000"+ + "\u0000\u01a4\u01a3\u0001\u0000\u0000\u0000\u01a5C\u0001\u0000\u0000\u0000"+ + "\u01a6\u01d1\u0005,\u0000\u0000\u01a7\u01a8\u0003h4\u0000\u01a8\u01a9"+ + "\u0005C\u0000\u0000\u01a9\u01d1\u0001\u0000\u0000\u0000\u01aa\u01d1\u0003"+ + "f3\u0000\u01ab\u01d1\u0003h4\u0000\u01ac\u01d1\u0003b1\u0000\u01ad\u01d1"+ + "\u0003F#\u0000\u01ae\u01d1\u0003j5\u0000\u01af\u01b0\u0005A\u0000\u0000"+ + "\u01b0\u01b5\u0003d2\u0000\u01b1\u01b2\u0005!\u0000\u0000\u01b2\u01b4"+ + "\u0003d2\u0000\u01b3\u01b1\u0001\u0000\u0000\u0000\u01b4\u01b7\u0001\u0000"+ + "\u0000\u0000\u01b5\u01b3\u0001\u0000\u0000\u0000\u01b5\u01b6\u0001\u0000"+ + "\u0000\u0000\u01b6\u01b8\u0001\u0000\u0000\u0000\u01b7\u01b5\u0001\u0000"+ + "\u0000\u0000\u01b8\u01b9\u0005B\u0000\u0000\u01b9\u01d1\u0001\u0000\u0000"+ + "\u0000\u01ba\u01bb\u0005A\u0000\u0000\u01bb\u01c0\u0003b1\u0000\u01bc"+ + "\u01bd\u0005!\u0000\u0000\u01bd\u01bf\u0003b1\u0000\u01be\u01bc\u0001"+ + "\u0000\u0000\u0000\u01bf\u01c2\u0001\u0000\u0000\u0000\u01c0\u01be\u0001"+ + "\u0000\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000\u01c1\u01c3\u0001"+ + "\u0000\u0000\u0000\u01c2\u01c0\u0001\u0000\u0000\u0000\u01c3\u01c4\u0005"+ + "B\u0000\u0000\u01c4\u01d1\u0001\u0000\u0000\u0000\u01c5\u01c6\u0005A\u0000"+ + "\u0000\u01c6\u01cb\u0003j5\u0000\u01c7\u01c8\u0005!\u0000\u0000\u01c8"+ + "\u01ca\u0003j5\u0000\u01c9\u01c7\u0001\u0000\u0000\u0000\u01ca\u01cd\u0001"+ + "\u0000\u0000\u0000\u01cb\u01c9\u0001\u0000\u0000\u0000\u01cb\u01cc\u0001"+ + "\u0000\u0000\u0000\u01cc\u01ce\u0001\u0000\u0000\u0000\u01cd\u01cb\u0001"+ + "\u0000\u0000\u0000\u01ce\u01cf\u0005B\u0000\u0000\u01cf\u01d1\u0001\u0000"+ + "\u0000\u0000\u01d0\u01a6\u0001\u0000\u0000\u0000\u01d0\u01a7\u0001\u0000"+ + "\u0000\u0000\u01d0\u01aa\u0001\u0000\u0000\u0000\u01d0\u01ab\u0001\u0000"+ + "\u0000\u0000\u01d0\u01ac\u0001\u0000\u0000\u0000\u01d0\u01ad\u0001\u0000"+ + "\u0000\u0000\u01d0\u01ae\u0001\u0000\u0000\u0000\u01d0\u01af\u0001\u0000"+ + "\u0000\u0000\u01d0\u01ba\u0001\u0000\u0000\u0000\u01d0\u01c5\u0001\u0000"+ + "\u0000\u0000\u01d1E\u0001\u0000\u0000\u0000\u01d2\u01d5\u0005/\u0000\u0000"+ + "\u01d3\u01d5\u0005@\u0000\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d4"+ + "\u01d3\u0001\u0000\u0000\u0000\u01d5G\u0001\u0000\u0000\u0000\u01d6\u01d9"+ + "\u0003@ \u0000\u01d7\u01d9\u0003F#\u0000\u01d8\u01d6\u0001\u0000\u0000"+ + "\u0000\u01d8\u01d7\u0001\u0000\u0000\u0000\u01d9I\u0001\u0000\u0000\u0000"+ + "\u01da\u01db\u0005\t\u0000\u0000\u01db\u01dc\u0005\u001a\u0000\u0000\u01dc"+ + "K\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\u000e\u0000\u0000\u01de\u01e3"+ + "\u0003N\'\u0000\u01df\u01e0\u0005!\u0000\u0000\u01e0\u01e2\u0003N\'\u0000"+ + "\u01e1\u01df\u0001\u0000\u0000\u0000\u01e2\u01e5\u0001\u0000\u0000\u0000"+ + "\u01e3\u01e1\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001\u0000\u0000\u0000"+ + "\u01e4M\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001\u0000\u0000\u0000\u01e6"+ + "\u01e8\u0003\n\u0005\u0000\u01e7\u01e9\u0007\u0004\u0000\u0000\u01e8\u01e7"+ + "\u0001\u0000\u0000\u0000\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9\u01ec"+ + "\u0001\u0000\u0000\u0000\u01ea\u01eb\u0005-\u0000\u0000\u01eb\u01ed\u0007"+ + "\u0005\u0000\u0000\u01ec\u01ea\u0001\u0000\u0000\u0000\u01ec\u01ed\u0001"+ + "\u0000\u0000\u0000\u01edO\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005\b"+ + "\u0000\u0000\u01ef\u01f0\u0003>\u001f\u0000\u01f0Q\u0001\u0000\u0000\u0000"+ + "\u01f1\u01f2\u0005\u0002\u0000\u0000\u01f2\u01f3\u0003>\u001f\u0000\u01f3"+ + "S\u0001\u0000\u0000\u0000\u01f4\u01f5\u0005\u000b\u0000\u0000\u01f5\u01fa"+ + "\u0003V+\u0000\u01f6\u01f7\u0005!\u0000\u0000\u01f7\u01f9\u0003V+\u0000"+ + "\u01f8\u01f6\u0001\u0000\u0000\u0000\u01f9\u01fc\u0001\u0000\u0000\u0000"+ + "\u01fa\u01f8\u0001\u0000\u0000\u0000\u01fa\u01fb\u0001\u0000\u0000\u0000"+ + "\u01fbU\u0001\u0000\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000\u01fd"+ + "\u01fe\u0003<\u001e\u0000\u01fe\u01ff\u0005T\u0000\u0000\u01ff\u0200\u0003"+ + "<\u001e\u0000\u0200W\u0001\u0000\u0000\u0000\u0201\u0202\u0005\u0001\u0000"+ + "\u0000\u0202\u0203\u0003\u0014\n\u0000\u0203\u0205\u0003j5\u0000\u0204"+ + "\u0206\u0003^/\u0000\u0205\u0204\u0001\u0000\u0000\u0000\u0205\u0206\u0001"+ + "\u0000\u0000\u0000\u0206Y\u0001\u0000\u0000\u0000\u0207\u0208\u0005\u0007"+ + "\u0000\u0000\u0208\u0209\u0003\u0014\n\u0000\u0209\u020a\u0003j5\u0000"+ + "\u020a[\u0001\u0000\u0000\u0000\u020b\u020c\u0005\n\u0000\u0000\u020c"+ + "\u020d\u0003:\u001d\u0000\u020d]\u0001\u0000\u0000\u0000\u020e\u0213\u0003"+ + "`0\u0000\u020f\u0210\u0005!\u0000\u0000\u0210\u0212\u0003`0\u0000\u0211"+ + "\u020f\u0001\u0000\u0000\u0000\u0212\u0215\u0001\u0000\u0000\u0000\u0213"+ + "\u0211\u0001\u0000\u0000\u0000\u0213\u0214\u0001\u0000\u0000\u0000\u0214"+ + "_\u0001\u0000\u0000\u0000\u0215\u0213\u0001\u0000\u0000\u0000\u0216\u0217"+ + "\u0003@ \u0000\u0217\u0218\u0005\u001f\u0000\u0000\u0218\u0219\u0003D"+ + "\"\u0000\u0219a\u0001\u0000\u0000\u0000\u021a\u021b\u0007\u0006\u0000"+ + "\u0000\u021bc\u0001\u0000\u0000\u0000\u021c\u021f\u0003f3\u0000\u021d"+ + "\u021f\u0003h4\u0000\u021e\u021c\u0001\u0000\u0000\u0000\u021e\u021d\u0001"+ + "\u0000\u0000\u0000\u021fe\u0001\u0000\u0000\u0000\u0220\u0222\u0007\u0000"+ + "\u0000\u0000\u0221\u0220\u0001\u0000\u0000\u0000\u0221\u0222\u0001\u0000"+ + "\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000\u0223\u0224\u0005\u001b"+ + "\u0000\u0000\u0224g\u0001\u0000\u0000\u0000\u0225\u0227\u0007\u0000\u0000"+ + "\u0000\u0226\u0225\u0001\u0000\u0000\u0000\u0226\u0227\u0001\u0000\u0000"+ + "\u0000\u0227\u0228\u0001\u0000\u0000\u0000\u0228\u0229\u0005\u001a\u0000"+ + "\u0000\u0229i\u0001\u0000\u0000\u0000\u022a\u022b\u0005\u0019\u0000\u0000"+ + "\u022bk\u0001\u0000\u0000\u0000\u022c\u022d\u0007\u0007\u0000\u0000\u022d"+ + "m\u0001\u0000\u0000\u0000\u022e\u022f\u0005\u0005\u0000\u0000\u022f\u0230"+ + "\u0003p8\u0000\u0230o\u0001\u0000\u0000\u0000\u0231\u0232\u0005A\u0000"+ + "\u0000\u0232\u0233\u0003\u0002\u0001\u0000\u0233\u0234\u0005B\u0000\u0000"+ + "\u0234q\u0001\u0000\u0000\u0000\u0235\u0236\u0005\r\u0000\u0000\u0236"+ + "\u0237\u0005d\u0000\u0000\u0237s\u0001\u0000\u0000\u0000\u0238\u0239\u0005"+ + "\u0003\u0000\u0000\u0239\u023c\u0005Z\u0000\u0000\u023a\u023b\u0005X\u0000"+ + "\u0000\u023b\u023d\u0003<\u001e\u0000\u023c\u023a\u0001\u0000\u0000\u0000"+ + "\u023c\u023d\u0001\u0000\u0000\u0000\u023d\u0247\u0001\u0000\u0000\u0000"+ + "\u023e\u023f\u0005Y\u0000\u0000\u023f\u0244\u0003v;\u0000\u0240\u0241"+ + "\u0005!\u0000\u0000\u0241\u0243\u0003v;\u0000\u0242\u0240\u0001\u0000"+ + "\u0000\u0000\u0243\u0246\u0001\u0000\u0000\u0000\u0244\u0242\u0001\u0000"+ + "\u0000\u0000\u0244\u0245\u0001\u0000\u0000\u0000\u0245\u0248\u0001\u0000"+ + "\u0000\u0000\u0246\u0244\u0001\u0000\u0000\u0000\u0247\u023e\u0001\u0000"+ + "\u0000\u0000\u0247\u0248\u0001\u0000\u0000\u0000\u0248u\u0001\u0000\u0000"+ + "\u0000\u0249\u024a\u0003<\u001e\u0000\u024a\u024b\u0005\u001f\u0000\u0000"+ + "\u024b\u024d\u0001\u0000\u0000\u0000\u024c\u0249\u0001\u0000\u0000\u0000"+ + "\u024c\u024d\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000"+ + "\u024e\u024f\u0003<\u001e\u0000\u024fw\u0001\u0000\u0000\u0000\u0250\u0251"+ + "\u0005\u0012\u0000\u0000\u0251\u0252\u0003$\u0012\u0000\u0252\u0253\u0005"+ + "X\u0000\u0000\u0253\u0254\u0003>\u001f\u0000\u0254y\u0001\u0000\u0000"+ + "\u0000\u0255\u0256\u0005\u0011\u0000\u0000\u0256\u0259\u00036\u001b\u0000"+ + "\u0257\u0258\u0005\u001c\u0000\u0000\u0258\u025a\u0003\u001e\u000f\u0000"+ + "\u0259\u0257\u0001\u0000\u0000\u0000\u0259\u025a\u0001\u0000\u0000\u0000"+ + "\u025a{\u0001\u0000\u0000\u0000;\u0087\u0090\u00a2\u00ae\u00b7\u00bf\u00c5"+ + "\u00cd\u00cf\u00d4\u00db\u00e0\u00eb\u00f1\u00f9\u00fb\u0106\u010d\u0118"+ + "\u011b\u0121\u012d\u0133\u013d\u0141\u0146\u0150\u0158\u0165\u0169\u016d"+ + "\u0174\u0178\u017f\u0186\u018d\u0195\u019d\u01a4\u01b5\u01c0\u01cb\u01d0"+ + "\u01d4\u01d8\u01e3\u01e8\u01ec\u01fa\u0205\u0213\u021e\u0221\u0226\u023c"+ + "\u0244\u0247\u024c\u0259"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 027281d44b2dc..556a97657635a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -332,6 +332,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterFunctionName(EsqlBaseParser.FunctionNameContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { } /** * {@inheritDoc} * @@ -500,6 +512,30 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterAggFields(EsqlBaseParser.AggFieldsContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitAggFields(EsqlBaseParser.AggFieldsContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterAggField(EsqlBaseParser.AggFieldContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitAggField(EsqlBaseParser.AggFieldContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index 463414ab67ea2..56b6999615f50 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -202,6 +202,13 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -300,6 +307,20 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitAggFields(EsqlBaseParser.AggFieldsContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitAggField(EsqlBaseParser.AggFieldContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index 1747ff001e162..cf658c4a73141 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -313,6 +313,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#functionName}. + * @param ctx the parse tree + */ + void enterFunctionName(EsqlBaseParser.FunctionNameContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#functionName}. + * @param ctx the parse tree + */ + void exitFunctionName(EsqlBaseParser.FunctionNameContext ctx); /** * Enter a parse tree produced by the {@code toDataType} * labeled alternative in {@link EsqlBaseParser#dataType}. @@ -455,6 +465,26 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitStatsCommand(EsqlBaseParser.StatsCommandContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#aggFields}. + * @param ctx the parse tree + */ + void enterAggFields(EsqlBaseParser.AggFieldsContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#aggFields}. + * @param ctx the parse tree + */ + void exitAggFields(EsqlBaseParser.AggFieldsContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#aggField}. + * @param ctx the parse tree + */ + void enterAggField(EsqlBaseParser.AggFieldContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#aggField}. + * @param ctx the parse tree + */ + void exitAggField(EsqlBaseParser.AggFieldContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedName}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index b7411d0f99c09..86c1d1aafc33a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -193,6 +193,12 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#functionName}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFunctionName(EsqlBaseParser.FunctionNameContext ctx); /** * Visit a parse tree produced by the {@code toDataType} * labeled alternative in {@link EsqlBaseParser#dataType}. @@ -278,6 +284,18 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#aggFields}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitAggFields(EsqlBaseParser.AggFieldsContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#aggField}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitAggField(EsqlBaseParser.AggFieldContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedName}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 5696ccea188b1..620a25e0170ea 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -98,10 +98,8 @@ private class PostProcessor extends EsqlBaseParserBaseListener { @Override public void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { // TODO remove this at some point - EsqlBaseParser.IdentifierOrParameterContext identifierOrParameter = ctx.identifierOrParameter(); - EsqlBaseParser.IdentifierContext idCtx = identifierOrParameter.identifier(); - String functionName = idCtx != null ? idCtx.getText() : identifierOrParameter.parameter().getText(); - if ("is_null".equalsIgnoreCase(functionName)) { + EsqlBaseParser.FunctionNameContext identifier = ctx.functionName(); + if (identifier.getText().equalsIgnoreCase("is_null")) { throw new ParsingException( source(ctx), "is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 50cf4bc998679..7ff09c23a1403 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar; +import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; @@ -44,6 +45,7 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionResolutionStrategy; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; +import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -593,13 +595,7 @@ public UnresolvedAttribute visitDereference(EsqlBaseParser.DereferenceContext ct @Override public Expression visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { - EsqlBaseParser.IdentifierOrParameterContext identifierOrParameter = ctx.identifierOrParameter(); - String name; - if (identifierOrParameter.identifier() != null) { - name = visitIdentifier(identifierOrParameter.identifier()); - } else { - name = unresolvedAttributeNameInParam(identifierOrParameter.parameter(), expression(identifierOrParameter.parameter())); - } + String name = visitFunctionName(ctx.functionName()); List args = expressions(ctx.booleanExpression()); if ("is_null".equals(EsqlFunctionRegistry.normalizeName(name))) { throw new ParsingException( @@ -616,6 +612,23 @@ public Expression visitFunctionExpression(EsqlBaseParser.FunctionExpressionConte return new UnresolvedFunction(source(ctx), name, FunctionResolutionStrategy.DEFAULT, args); } + @Override + public String visitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { + if (ctx.MATCH() != null) { + return ctx.MATCH().getText(); + } + return visitIdentifierOrParameter(ctx.identifierOrParameter()); + } + + @Override + public String visitIdentifierOrParameter(EsqlBaseParser.IdentifierOrParameterContext ctx) { + if (ctx.identifier() != null) { + return visitIdentifier(ctx.identifier()); + } + + return unresolvedAttributeNameInParam(ctx.parameter(), expression(ctx.parameter())); + } + @Override public Expression visitInlineCast(EsqlBaseParser.InlineCastContext ctx) { Source source = source(ctx); @@ -731,9 +744,12 @@ private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternConte @Override public Alias visitField(EsqlBaseParser.FieldContext ctx) { + return visitField(ctx, source(ctx)); + } + + private Alias visitField(EsqlBaseParser.FieldContext ctx, Source source) { UnresolvedAttribute id = visitQualifiedName(ctx.qualifiedName()); Expression value = expression(ctx.booleanExpression()); - var source = source(ctx); String name = id == null ? source.text() : id.name(); return new Alias(source, name, value); } @@ -743,6 +759,36 @@ public List visitFields(EsqlBaseParser.FieldsContext ctx) { return ctx != null ? visitList(this, ctx.field(), Alias.class) : new ArrayList<>(); } + @Override + public NamedExpression visitAggField(EsqlBaseParser.AggFieldContext ctx) { + Source source = source(ctx); + Alias field = visitField(ctx.field(), source); + var filterExpression = ctx.booleanExpression(); + + if (filterExpression != null) { + Expression condition = expression(filterExpression); + Expression child = field.child(); + // basic check as the filter can be specified only on a function (should be an aggregate but we can't determine that yet) + if (field.child().anyMatch(Function.class::isInstance)) { + field = field.replaceChild(new FilteredExpression(field.source(), child, condition)); + } + // allow condition only per aggregated function + else { + throw new ParsingException( + condition.source(), + "WHERE clause allowed only for aggregate functions [{}]", + field.sourceText() + ); + } + } + return field; + } + + @Override + public List visitAggFields(EsqlBaseParser.AggFieldsContext ctx) { + return ctx != null ? visitList(this, ctx.aggField(), Alias.class) : new ArrayList<>(); + } + /** * Similar to {@link #visitFields(EsqlBaseParser.FieldsContext)} however avoids wrapping the expression * into an Alias. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index c90c3cba4ef24..dc913cd2f14f4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -298,13 +298,12 @@ public PlanFactory visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { return input -> new Aggregate(source(ctx), input, Aggregate.AggregateType.STANDARD, stats.groupings, stats.aggregates); } - private record Stats(List groupings, List aggregates) { + private record Stats(List groupings, List aggregates) {} - } - - private Stats stats(Source source, EsqlBaseParser.FieldsContext groupingsCtx, EsqlBaseParser.FieldsContext aggregatesCtx) { + private Stats stats(Source source, EsqlBaseParser.FieldsContext groupingsCtx, EsqlBaseParser.AggFieldsContext aggregatesCtx) { List groupings = visitGrouping(groupingsCtx); - List aggregates = new ArrayList<>(visitFields(aggregatesCtx)); + List aggregates = new ArrayList<>(visitAggFields(aggregatesCtx)); + if (aggregates.isEmpty() && groupings.isEmpty()) { throw new ParsingException(source, "At least one aggregation or grouping expression required in [{}]", source.text()); } @@ -341,9 +340,11 @@ public PlanFactory visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandCont if (false == EsqlPlugin.INLINESTATS_FEATURE_FLAG.isEnabled()) { throw new ParsingException(source(ctx), "INLINESTATS command currently requires a snapshot build"); } - List aggregates = new ArrayList<>(visitFields(ctx.stats)); + List aggFields = visitAggFields(ctx.stats); + List aggregates = new ArrayList<>(aggFields); List groupings = visitGrouping(ctx.grouping); aggregates.addAll(groupings); + // TODO: add support for filters return input -> new InlineStats(source(ctx), input, new ArrayList<>(groupings), aggregates); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java index 8445c8236c45a..3b7240dcd693b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java @@ -59,6 +59,7 @@ static AggregateType readType(StreamInput in) throws IOException { private final AggregateType aggregateType; private final List groupings; private final List aggregates; + private List lazyOutput; public Aggregate( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java index 0e71963e29270..94a9246a56f83 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java @@ -10,10 +10,12 @@ import org.elasticsearch.compute.aggregation.Aggregator; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.AggregatorMode; +import org.elasticsearch.compute.aggregation.FilteredAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.GroupingAggregator; import org.elasticsearch.compute.aggregation.blockhash.BlockHash; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.operator.AggregationOperator; +import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.HashAggregationOperator.HashAggregationOperatorFactory; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -24,6 +26,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.evaluator.EvalMapper; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; @@ -231,11 +234,14 @@ private void aggregatesToFactory( boolean grouping, Consumer consumer ) { + // extract filtering channels - and wrap the aggregation with the new evaluator expression only during the init phase for (NamedExpression ne : aggregates) { + // a filter can only appear on aggregate function, not on the grouping columns + if (ne instanceof Alias alias) { var child = alias.child(); if (child instanceof AggregateFunction aggregateFunction) { - List sourceAttr; + List sourceAttr = new ArrayList<>(); if (mode == AggregatorMode.INITIAL) { // TODO: this needs to be made more reliable - use casting to blow up when dealing with expressions (e+1) @@ -251,19 +257,22 @@ private void aggregatesToFactory( ); } } else { - sourceAttr = aggregateFunction.inputExpressions().stream().map(e -> { - Attribute attr = Expressions.attribute(e); + // extra dependencies like TS ones (that require a timestamp) + for (Expression input : aggregateFunction.references()) { + Attribute attr = Expressions.attribute(input); if (attr == null) { throw new EsqlIllegalArgumentException( "Cannot work with target field [{}] for agg [{}]", - e.sourceText(), + input.sourceText(), aggregateFunction.sourceText() ); } - return attr; - }).toList(); + sourceAttr.add(attr); + } } - } else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) { + } + // coordinator/exchange phase + else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) { if (grouping) { sourceAttr = aggregateMapper.mapGrouping(aggregateFunction); } else { @@ -274,16 +283,27 @@ private void aggregatesToFactory( } List inputChannels = sourceAttr.stream().map(attr -> layout.get(attr.id()).channel()).toList(); assert inputChannels.stream().allMatch(i -> i >= 0) : inputChannels; - if (aggregateFunction instanceof ToAggregator agg) { - consumer.accept(new AggFunctionSupplierContext(agg.supplier(inputChannels), mode)); - } else { - throw new EsqlIllegalArgumentException("aggregate functions must extend ToAggregator"); + + AggregatorFunctionSupplier aggSupplier = supplier(aggregateFunction, inputChannels); + + // apply the filter only in the initial phase - as the rest of the data is already filtered + if (aggregateFunction.hasFilter() && mode.isInputPartial() == false) { + EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator(aggregateFunction.filter(), layout); + aggSupplier = new FilteredAggregatorFunctionSupplier(aggSupplier, evalFactory); } + consumer.accept(new AggFunctionSupplierContext(aggSupplier, mode)); } } } } + private static AggregatorFunctionSupplier supplier(AggregateFunction aggregateFunction, List inputChannels) { + if (aggregateFunction instanceof ToAggregator delegate) { + return delegate.supplier(inputChannels); + } + throw new EsqlIllegalArgumentException("aggregate functions must extend ToAggregator"); + } + private record GroupSpec(Integer channel, Attribute attribute) { BlockHash.GroupSpec toHashGroupSpec() { if (channel == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 13ce9ba77cc71..c322135198262 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -98,39 +98,39 @@ private record AggDef(Class aggClazz, String type, String extra, boolean grou .collect(Collectors.toUnmodifiableMap(aggDef -> aggDef, AggregateMapper::lookupIntermediateState)); /** Cache of aggregates to intermediate expressions. */ - private final HashMap> cache; + private final HashMap> cache; AggregateMapper() { cache = new HashMap<>(); } - public List mapNonGrouping(List aggregates) { + public List mapNonGrouping(List aggregates) { return doMapping(aggregates, false); } - public List mapNonGrouping(Expression aggregate) { + public List mapNonGrouping(Expression aggregate) { return map(aggregate, false).toList(); } - public List mapGrouping(List aggregates) { + public List mapGrouping(List aggregates) { return doMapping(aggregates, true); } - private List doMapping(List aggregates, boolean grouping) { + private List doMapping(List aggregates, boolean grouping) { AttributeMap attrToExpressions = new AttributeMap<>(); aggregates.stream().flatMap(agg -> map(agg, grouping)).forEach(ne -> attrToExpressions.put(ne.toAttribute(), ne)); return attrToExpressions.values().stream().toList(); } - public List mapGrouping(Expression aggregate) { + public List mapGrouping(Expression aggregate) { return map(aggregate, true).toList(); } - private Stream map(Expression aggregate, boolean grouping) { + private Stream map(Expression aggregate, boolean grouping) { return cache.computeIfAbsent(Alias.unwrap(aggregate), aggKey -> computeEntryForAgg(aggKey, grouping)).stream(); } - private static List computeEntryForAgg(Expression aggregate, boolean grouping) { + private static List computeEntryForAgg(Expression aggregate, boolean grouping) { var aggDef = aggDefOrNull(aggregate, grouping); if (aggDef != null) { var is = getNonNull(aggDef); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 18aa2628fdc7c..2c8604a7c4a80 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -24,15 +24,18 @@ import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators; import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler; import org.elasticsearch.xpack.esql.core.querydsl.query.MatchAll; +import org.elasticsearch.xpack.esql.core.querydsl.query.MatchQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; -import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils; @@ -55,6 +58,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf; @@ -85,17 +89,11 @@ public final class EsqlExpressionTranslators { new ExpressionTranslators.StringQueries(), new ExpressionTranslators.Matches(), new ExpressionTranslators.MultiMatches(), - new FullTextFunctions(), + new MatchFunctionTranslator(), + new QueryStringFunctionTranslator(), new Scalars() ); - public static class FullTextFunctions extends ExpressionTranslator { - @Override - protected Query asQuery(FullTextFunction fullTextFunction, TranslatorHandler handler) { - return fullTextFunction.asQuery(); - } - } - public static Query toQuery(Expression e, TranslatorHandler handler) { Query translation = null; for (ExpressionTranslator translator : QUERY_TRANSLATORS) { @@ -528,4 +526,18 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) { ); } } + + public static class MatchFunctionTranslator extends ExpressionTranslator { + @Override + protected Query asQuery(Match match, TranslatorHandler handler) { + return new MatchQuery(match.source(), ((FieldAttribute) match.field()).name(), match.queryAsText()); + } + } + + public static class QueryStringFunctionTranslator extends ExpressionTranslator { + @Override + protected Query asQuery(QueryString queryString, TranslatorHandler handler) { + return new QueryStringQuery(queryString.source(), queryString.queryAsText(), Map.of(), null); + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 965358c0c3f8c..ce072e7b0a438 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -67,10 +67,7 @@ import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; -import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext; -import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.TestLocalPhysicalPlanOptimizer; -import org.elasticsearch.xpack.esql.optimizer.TestPhysicalPlanOptimizer; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -167,7 +164,6 @@ public class CsvTests extends ESTestCase { private final EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); private final EsqlParser parser = new EsqlParser(); private final Mapper mapper = new Mapper(functionRegistry); - private final PhysicalPlanOptimizer physicalPlanOptimizer = new TestPhysicalPlanOptimizer(new PhysicalOptimizerContext(configuration)); private ThreadPool threadPool; private Executor executor; @@ -252,6 +248,10 @@ public final void test() throws Throwable { "can't use QSTR function in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName()) ); + assumeFalse( + "can't use MATCH function in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_FUNCTION.capabilityName()) + ); if (Build.current().isSnapshot()) { assertThat( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 5d75549893512..d365ee3bb2e51 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1219,7 +1219,7 @@ public void testAggsOverGroupingKey() throws Exception { assertThat(output, hasSize(2)); var aggs = agg.aggregates(); var min = as(Alias.unwrap(aggs.get(0)), Min.class); - assertThat(min.arguments(), hasSize(1)); + assertThat(min.arguments(), hasSize(2)); // field + filter var group = Alias.unwrap(agg.groupings().get(0)); assertEquals(min.arguments().get(0), group); } @@ -1241,7 +1241,7 @@ public void testAggsOverGroupingKeyWithAlias() throws Exception { assertThat(output, hasSize(2)); var aggs = agg.aggregates(); var min = as(Alias.unwrap(aggs.get(0)), Min.class); - assertThat(min.arguments(), hasSize(1)); + assertThat(min.arguments(), hasSize(2)); // field + filter assertEquals(Expressions.attribute(min.arguments().get(0)), Expressions.attribute(agg.groupings().get(0))); } @@ -1638,7 +1638,7 @@ public void testCounterTypes() { var attributes = limit.output().stream().collect(Collectors.toMap(NamedExpression::name, a -> a)); assertThat( attributes.keySet(), - equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in")) + equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in", "network.message_out")) ); assertThat(attributes.get("network.connections").dataType(), equalTo(DataType.LONG)); assertThat(attributes.get("network.bytes_in").dataType(), equalTo(DataType.COUNTER_LONG)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 83b3e88c57964..63f7629f3c720 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.Build; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -25,12 +26,16 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsConstant; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -355,6 +360,40 @@ public void testAggsInsideGrouping() { ); } + public void testAggFilterOnNonAggregates() { + assertEquals( + "1:36: WHERE clause allowed only for aggregate functions, none found in [emp_no + 1 where languages > 1]", + error("from test | stats emp_no + 1 where languages > 1 by emp_no") + ); + assertEquals( + "1:53: WHERE clause allowed only for aggregate functions, none found in [abs(emp_no + languages) % 2 WHERE languages > 1]", + error("from test | stats abs(emp_no + languages) % 2 WHERE languages > 1 by emp_no, languages") + ); + } + + public void testAggFilterOnBucketingOrAggFunctions() { + // query passes when the bucket function is part of the BY clause + query("from test | stats max(languages) WHERE bucket(salary, 10) > 1 by bucket(salary, 10)"); + + // but fails if it's different + assertEquals( + "1:40: can only use grouping function [bucket(salary, 10)] part of the BY clause", + error("from test | stats max(languages) WHERE bucket(salary, 10) > 1 by emp_no") + ); + + assertEquals( + "1:40: cannot use aggregate function [max(salary)] in aggregate WHERE clause [max(languages) WHERE max(salary) > 1]", + error("from test | stats max(languages) WHERE max(salary) > 1 by emp_no") + ); + + assertEquals( + "1:40: cannot use aggregate function [max(salary)] in aggregate WHERE clause [max(languages) WHERE max(salary) + 2 > 1]", + error("from test | stats max(languages) WHERE max(salary) + 2 > 1 by emp_no") + ); + + assertEquals("1:60: Unknown column [m]", error("from test | stats m = max(languages), min(languages) WHERE m + 2 > 1 by emp_no")); + } + public void testGroupingInsideAggsAsAgg() { assertEquals( "1:18: can only use grouping function [bucket(emp_no, 5.)] part of the BY clause", @@ -908,6 +947,25 @@ public void testSpatialSort() { assertEquals("1:42: cannot sort on cartesian_shape", error("FROM countries_bbox_web | LIMIT 5 | sort shape", countriesBboxWeb)); } + public void testSourceSorting() { + assertEquals("1:35: cannot sort on _source", error("from test metadata _source | sort _source")); + } + + public void testCountersSorting() { + Map counterDataTypes = Map.of( + COUNTER_DOUBLE, + "network.message_in", + COUNTER_INTEGER, + "network.message_out", + COUNTER_LONG, + "network.bytes_out" + ); + for (DataType counterDT : counterDataTypes.keySet()) { + var fieldName = counterDataTypes.get(counterDT); + assertEquals("1:18: cannot sort on " + counterDT.name().toLowerCase(Locale.ROOT), error("from test | sort " + fieldName, tsdb)); + } + } + public void testInlineImpossibleConvert() { assertEquals("1:5: argument of [false::ip] must be [ip or string], found value [false] type [boolean]", error("ROW false::ip")); } @@ -1086,9 +1144,14 @@ public void testMatchFilter() throws Exception { ); } - public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { - assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testMatchFunctionNotAllowedAfterCommands() throws Exception { + assertEquals( + "1:24: [MATCH] function cannot be used after LIMIT", + error("from test | limit 10 | where match(first_name, \"Anna\")") + ); + } + public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { // Source commands assertEquals("1:13: [QSTR] function cannot be used after SHOW", error("show info | where qstr(\"8.16.0\")")); assertEquals("1:17: [QSTR] function cannot be used after ROW", error("row a= \"Anna\" | where qstr(\"Anna\")")); @@ -1146,26 +1209,145 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { ); } - public void testQueryStringFunctionsOnlyAllowedInWhere() throws Exception { - assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { + assertEquals("1:9: [QSTR] function is only supported in WHERE commands", error("row a = qstr(\"Anna\")")); + checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")"); + } + + public void testMatchFunctionOnlyAllowedInWhere() throws Exception { + checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")"); + } - assertEquals("1:22: [QSTR] function is only supported in WHERE commands", error("from test | eval y = qstr(\"Anna\")")); - assertEquals("1:18: [QSTR] function is only supported in WHERE commands", error("from test | sort qstr(\"Connection\") asc")); - assertEquals("1:5: [QSTR] function is only supported in WHERE commands", error("row qstr(\"Connection\")")); + private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation) throws Exception { + assertEquals( + "1:22: [" + functionName + "] function is only supported in WHERE commands", + error("from test | eval y = " + functionInvocation) + ); + assertEquals( + "1:18: [" + functionName + "] function is only supported in WHERE commands", + error("from test | sort " + functionInvocation + " asc") + ); assertEquals( - "1:23: [QSTR] function is only supported in WHERE commands", - error("from test | STATS c = qstr(\"foo\") BY languages") + "1:23: [" + functionName + "] function is only supported in WHERE commands", + error("from test | STATS c = " + functionInvocation + " BY first_name") ); } public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { - assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + assertEquals( + "1:19: argument of [qstr(first_name)] must be a constant, received [first_name]", + error("from test | where qstr(first_name)") + ); + assertEquals("1:19: argument of [qstr(null)] cannot be null, received [null]", error("from test | where qstr(null)")); + // Other value types are tested in QueryStringFunctionTests + } - assertEquals("1:19: argument of [QSTR] must be a constant, received [first_name]", error("from test | where qstr(first_name)")); - assertEquals("1:19: argument of [QSTR] cannot be null, received [null]", error("from test | where qstr(null)")); + public void testQueryStringWithDisjunctions() { + checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")"); + } + + public void testMatchWithDisjunctions() { + checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")"); + } + + private void checkWithDisjunctions(String functionName, String functionInvocation) { + assertEquals( + LoggerMessageFormat.format( + null, + "1:19: Invalid condition [{} or length(first_name) > 12]. " + "Function {} can't be used as part of an or condition", + functionInvocation, + functionName + ), + error("from test | where " + functionInvocation + " or length(first_name) > 12") + ); + assertEquals( + LoggerMessageFormat.format( + null, + "1:19: Invalid condition [({} and first_name is not null) or (length(first_name) > 12 and first_name is null)]. " + + "Function {} can't be used as part of an or condition", + functionInvocation, + functionName + ), + error( + "from test | where (" + + functionInvocation + + " and first_name is not null) or (length(first_name) > 12 and first_name is null)" + ) + ); + assertEquals( + LoggerMessageFormat.format( + null, + "1:19: Invalid condition [({} and first_name is not null) or first_name is null]. " + + "Function {} can't be used as part of an or condition", + functionInvocation, + functionName + ), + error("from test | where (" + functionInvocation + " and first_name is not null) or first_name is null") + ); + } + + public void testQueryStringFunctionWithNonBooleanFunctions() { + checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")"); + } + + public void testMatchFunctionWithNonBooleanFunctions() { + checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")"); + } + + private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, String functionInvocation) { + assertEquals( + "1:19: Invalid condition [" + functionInvocation + " is not null]. Function " + functionName + " can't be used with ISNOTNULL", + error("from test | where " + functionInvocation + " is not null") + ); + assertEquals( + "1:19: Invalid condition [" + functionInvocation + " is null]. Function " + functionName + " can't be used with ISNULL", + error("from test | where " + functionInvocation + " is null") + ); + assertEquals( + "1:19: Invalid condition [" + + functionInvocation + + " in (\"hello\", \"world\")]. Function " + + functionName + + " can't be used with IN", + error("from test | where " + functionInvocation + " in (\"hello\", \"world\")") + ); + } + + public void testMatchFunctionArgNotConstant() throws Exception { + assertEquals( + "1:19: second argument of [match(first_name, first_name)] must be a constant, received [first_name]", + error("from test | where match(first_name, first_name)") + ); + assertEquals( + "1:59: second argument of [match(first_name, query)] must be a constant, received [query]", + error("from test | eval query = concat(\"first\", \" name\") | where match(first_name, query)") + ); // Other value types are tested in QueryStringFunctionTests } + // These should pass eventually once we lift some restrictions on match function + public void testMatchFunctionCurrentlyUnsupportedBehaviour() throws Exception { + assertEquals( + "1:68: Unknown column [first_name]", + error("from test | stats max_salary = max(salary) by emp_no | where match(first_name, \"Anna\")") + ); + } + + public void testMatchFunctionNullArgs() throws Exception { + assertEquals( + "1:19: first argument of [match(null, \"query\")] cannot be null, received [null]", + error("from test | where match(null, \"query\")") + ); + assertEquals( + "1:19: second argument of [match(first_name, null)] cannot be null, received [null]", + error("from test | where match(first_name, null)") + ); + } + + public void testMatchFunctionTargetsExistingField() throws Exception { + assertEquals("1:39: Unknown column [first_name]", error("from test | keep emp_no | where match(first_name, \"Anna\")")); + } + public void testCoalesceWithMixedNumericTypes() { assertEquals( "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]", @@ -1359,6 +1541,10 @@ public void testToDatePeriodToTimeDurationWithInvalidType() { ); } + private void query(String query) { + defaultAnalyzer.analyze(parser.createStatement(query)); + } + private String error(String query) { return error(query, defaultAnalyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java index 94b2a81b308d7..ea7c480817317 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateSerializationTests.java @@ -36,4 +36,9 @@ protected Rate mutateInstance(Rate instance) throws IOException { } return new Rate(source, field, timestamp, unit); } + + @Override + protected boolean alwaysEmptySource() { + return true; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java index 82bf57d1a194e..e74b26c87c84f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopSerializationTests.java @@ -36,4 +36,9 @@ protected Top mutateInstance(Top instance) throws IOException { } return new Top(source, field, limit, order); } + + @Override + protected boolean alwaysEmptySource() { + return true; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java new file mode 100644 index 0000000000000..d37bc89635c1d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +@FunctionName("match") +public class MatchTests extends AbstractFunctionTestCase { + + public MatchTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + Set supported = Set.of(DataType.KEYWORD, DataType.TEXT); + List> supportedPerPosition = List.of(supported, supported); + List suppliers = new LinkedList<>(); + for (DataType fieldType : validStringDataTypes()) { + for (DataType queryType : validStringDataTypes()) { + suppliers.add( + new TestCaseSupplier( + "<" + fieldType + "-ES field, " + queryType + ">", + List.of(fieldType, queryType), + () -> testCase(fieldType, randomIdentifier(), queryType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) + ) + ); + suppliers.add( + new TestCaseSupplier( + "<" + fieldType + "-non ES field, " + queryType + ">", + List.of(fieldType, queryType), + typeErrorSupplier(true, supportedPerPosition, List.of(fieldType, queryType), MatchTests::matchTypeErrorSupplier) + ) + ); + } + } + List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests + return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); + } + + private static String matchTypeErrorSupplier(boolean includeOrdinal, List> validPerPosition, List types) { + return "[] cannot operate on [" + types.getFirst().typeName() + "], which is not a field from an index mapping"; + } + + private static List validStringDataTypes() { + return Arrays.stream(DataType.values()).filter(DataType::isString).toList(); + } + + private static TestCaseSupplier.TestCase testCase( + DataType fieldType, + String field, + DataType queryType, + String query, + Matcher matcher + ) { + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData( + new FieldExpression(field, List.of(new FieldExpression.FieldValue(field))), + fieldType, + "field" + ), + new TestCaseSupplier.TypedData(new BytesRef(query), queryType, "query") + ), + "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", + DataType.BOOLEAN, + matcher + ); + } + + @Override + protected Expression build(Source source, List args) { + return new Match(source, args.get(0), args.get(1)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java similarity index 84% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java index 37e16a2499cd9..2dfdb05ec8ecc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java @@ -18,25 +18,18 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; -import org.junit.BeforeClass; import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.function.Supplier; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.QSTR_FUNCTION; import static org.hamcrest.Matchers.equalTo; @FunctionName("qstr") -public class QueryStringFunctionTests extends AbstractFunctionTestCase { +public class QueryStringTests extends AbstractFunctionTestCase { - @BeforeClass - public static void checkFunctionEnabled() { - assumeTrue("QSTR capability should be enabled ", QSTR_FUNCTION.isEnabled()); - } - - public QueryStringFunctionTests(@Name("TestCase") Supplier testCaseSupplier) { + public QueryStringTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -77,6 +70,6 @@ private static TestCaseSupplier.TestCase testCase(DataType strType, String str, @Override protected Expression build(Source source, List args) { - return new QueryStringFunction(source, args.get(0)); + return new QueryString(source, args.get(0)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index c2779b7dbc46d..8501dd6e478df 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.EsqlTestUtils.TestSearchStats; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; @@ -50,7 +49,6 @@ import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; -import org.elasticsearch.xpack.esql.plan.physical.FilterExec; import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -385,7 +383,6 @@ public void testMultiCountAllWithFilter() { * \_EsQueryExec[test], indexMode[standard], query[{"query_string":{"query":"last_name: Smith","fields":[]}}] */ public void testQueryStringFunction() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); var plan = plannerOptimizer.plan(""" from test | where qstr("last_name: Smith") @@ -414,7 +411,6 @@ public void testQueryStringFunction() { * "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324] */ public void testQueryStringFunctionConjunctionWhereOperands() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); String queryText = """ from test | where qstr("last_name: Smith") and emp_no > 10010 @@ -438,20 +434,56 @@ public void testQueryStringFunctionConjunctionWhereOperands() { /** * Expecting * LimitExec[1000[INTEGER]] - * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] - * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8]] - * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen] - * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"should":[{"query_string":{"query":"last_name: Smith","fields":[]}}, - * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@2:37"}}], - * "boost":1.0}}][_doc{f}#13], limit[1000], sort[] estimatedRowSize[324] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{ + * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] */ - public void testQueryStringFunctionDisjunctionWhereClauses() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testQueryStringFunctionWithFunctionsPushedToLucene() { String queryText = """ from test - | where qstr("last_name: Smith") or emp_no > 10010 + | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") + """; + var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(terms); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162],false] + * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] + * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], + * query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], + * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionMultipleWhereClauses() { + String queryText = """ + from test + | where qstr("last_name: Smith") + | where emp_no > 10010 """; var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); @@ -462,34 +494,31 @@ public void testQueryStringFunctionDisjunctionWhereClauses() { var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - Source filterSource = new Source(2, 36, "emp_no > 10000"); + Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); - var expected = QueryBuilders.boolQuery().should(queryString).should(range); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } /** * Expecting * LimitExec[1000[INTEGER]] - * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ - * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], - * false] - * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ - * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] - * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] - * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{ - * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, - * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool": + * {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}] */ - public void testQueryStringFunctionWithFunctionsPushedToLucene() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testQueryStringFunctionMultipleQstrClauses() { String queryText = """ from test - | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") + | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]") """; - var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); - var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); var limit = as(plan, LimitExec.class); var exchange = as(limit.child(), ExchangeExec.class); @@ -498,45 +527,103 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() { var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); - var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); - var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(queryString).must(terms); + var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); + var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); + var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); assertThat(query.query().toString(), is(expected.toString())); } /** * Expecting - *LimitExec[1000[INTEGER]] - * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] - * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8]] - * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, gender{f}#5, job{f}#] - * \_LimitExec[1000[INTEGER]] - * \_FilterExec[LENGTH(first_name{f}#4) > 10[INTEGER]] - * \_FieldExtractExec[first_name{f}#4] - * \_EsQueryExec[test], indexMode[standard], - * query[{"query_string":{"query":"last_name: Smith","fields":[]}}][_doc{f}#13], limit[], sort[] estimatedRowSize[324] + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"match":{"last_name":{"query":"Smith"}}}] */ - public void testQueryStringFunctionWithFunctionNotPushedDown() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testMatchFunction() { + var plan = plannerOptimizer.plan(""" + from test + | where match(last_name, "Smith") + """, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + var expected = QueryBuilders.matchQuery("last_name", "Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418],false] + * \_ProjectExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418]] + * \_FieldExtractExec[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#] + * \EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}}, + * "source":"emp_no > 10010@2:39"}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] + */ + public void testMatchFunctionConjunctionWhereOperands() { String queryText = """ from test - | where qstr("last_name: Smith") and length(first_name) > 10 + | where match(last_name, "Smith") and emp_no > 10010 """; var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); - var firstLimit = as(plan, LimitExec.class); - var exchange = as(firstLimit.child(), ExchangeExec.class); + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); - var secondLimit = as(field.child(), LimitExec.class); - var filter = as(secondLimit.child(), FilterExec.class); - var fieldExtract = as(filter.child(), FieldExtractExec.class); - var query = as(fieldExtract.child(), EsQueryExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); - var expected = QueryBuilders.queryStringQuery("last_name: Smith"); + Source filterSource = new Source(2, 38, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.matchQuery("last_name", "Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"text":{"query":"beta"}}}, + * {"esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:33"}}],"boost":1.0}}][_doc{f}#22], limit[1000], sort[] estimatedRowSize[354] + */ + public void testMatchFunctionWithFunctionsPushedToLucene() { + String queryText = """ + from test + | where match(text, "beta") and cidr_match(ip, "127.0.0.1/32") + """; + var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var queryString = QueryBuilders.matchQuery("text", "beta"); + var expected = QueryBuilders.boolQuery().must(queryString).must(terms); assertThat(query.query().toString(), is(expected.toString())); } @@ -548,16 +635,14 @@ public void testQueryStringFunctionWithFunctionNotPushedDown() { * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] - * \_EsQueryExec[test], indexMode[standard], - * query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, - * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], - * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}}, + * "source":"emp_no > 10010@3:9"}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] */ - public void testQueryStringFunctionMultipleWhereClauses() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testMatchFunctionMultipleWhereClauses() { String queryText = """ from test - | where qstr("last_name: Smith") + | where match(last_name, "Smith") | where emp_no > 10010 """; var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); @@ -571,7 +656,7 @@ public void testQueryStringFunctionMultipleWhereClauses() { Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); - var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var queryString = QueryBuilders.matchQuery("last_name", "Smith"); var expected = QueryBuilders.boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -584,15 +669,13 @@ public void testQueryStringFunctionMultipleWhereClauses() { * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na * me{f}#6, long_noidx{f}#11, salary{f}#7]] * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] - * \_EsQueryExec[test], indexMode[standard], query[{"bool": - * {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, - * {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, + * {"match":{"first_name":{"query":"John"}}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] */ - public void testQueryStringFunctionMultipleQstrClauses() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testMatchFunctionMultipleQstrClauses() { String queryText = """ from test - | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]") + | where match(last_name, "Smith") and match(first_name, "John") """; var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); @@ -603,8 +686,8 @@ public void testQueryStringFunctionMultipleQstrClauses() { var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); - var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); + var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith"); + var queryStringRight = QueryBuilders.matchQuery("first_name", "John"); var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); assertThat(query.query().toString(), is(expected.toString())); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 6ff541de1854a..8d7c1997f78e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -537,6 +537,24 @@ public void testCombineProjectionWithDuplicateAggregation() { assertThat(Expressions.names(agg.groupings()), contains("last_name", "first_name")); } + /** + * Limit[1000[INTEGER]] + * \_Aggregate[STANDARD,[],[SUM(salary{f}#12,true[BOOLEAN]) AS sum(salary), SUM(salary{f}#12,last_name{f}#11 == [44 6f 65][KEYW + * ORD]) AS sum(salary) WheRe last_name == "Doe"]] + * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + */ + public void testStatsWithFilteringDefaultAliasing() { + var plan = plan(""" + from test + | stats sum(salary), sum(salary) WheRe last_name == "Doe" + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates(), hasSize(2)); + assertThat(Expressions.names(agg.aggregates()), contains("sum(salary)", "sum(salary) WheRe last_name == \"Doe\"")); + } + public void testQlComparisonOptimizationsApply() { var plan = plan(""" from test @@ -5565,6 +5583,38 @@ public void testToDatePeriodToTimeDurationWithField() { assertEquals("1:60: argument of [to_timeduration(x)] must be a constant, received [x]", e.getMessage().substring(header.length())); } + // These should pass eventually once we lift some restrictions on match function + public void testMatchWithNonIndexedColumnCurrentlyUnsupported() { + final String header = "Found 1 problem\nline "; + VerificationException e = expectThrows(VerificationException.class, () -> plan(""" + from test | eval initial = substring(first_name, 1) | where match(initial, "A")""")); + assertTrue(e.getMessage().startsWith("Found ")); + assertEquals( + "1:67: [MATCH] cannot operate on [initial], which is not a field from an index mapping", + e.getMessage().substring(header.length()) + ); + + e = expectThrows(VerificationException.class, () -> plan(""" + from test | eval text=concat(first_name, last_name) | where match(text, "cat")""")); + assertTrue(e.getMessage().startsWith("Found ")); + assertEquals( + "1:67: [MATCH] cannot operate on [text], which is not a field from an index mapping", + e.getMessage().substring(header.length()) + ); + } + + public void testMatchFunctionIsNotNullable() { + String queryText = """ + row n = null | eval text = n + 5 | where match(text::keyword, "Anna") + """; + + VerificationException ve = expectThrows(VerificationException.class, () -> plan(queryText)); + assertThat( + ve.getMessage(), + containsString("[MATCH] cannot operate on [text::keyword], which is not a field from an index mapping") + ); + } + private Literal nullOf(DataType dataType) { return new Literal(Source.EMPTY, null, dataType); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 6746b8ff61268..114aed68761fe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -3211,28 +3211,28 @@ public void testPushSpatialIntersectsStringToSource() { /** * Plan: * - * LimitExec[1000[INTEGER]] - * \_ExchangeExec[[],false] - * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[ + * EvalExec[[scalerank{f}#8 AS rank]] + * \_LimitExec[1000[INTEGER]] + * \_ExchangeExec[[],false] + * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[ * Limit[1000[INTEGER]] - * \_Filter[rank{r}#4 lt 4[INTEGER]] - * \_Eval[[scalerank{f}#8 AS rank]] - * \_EsRelation[airports][abbrev{f}#6, city{f}#12, city_location{f}#13, count..]]] + * \_Filter[scalerank{f}#8 < 4[INTEGER]] + * \_EsRelation[airports][abbrev{f}#6, city{f}#12, city_location{f}#13, count..]]] * * Optimized: * - * LimitExec[1000[INTEGER]] - * \_ExchangeExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8, - * type{f}#9, rank{r}#4],false] - * \_ProjectExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8, - * type{f}#9, rank{r}#4]] - * \_FieldExtractExec[abbrev{f}#6, city{f}#12, city_location{f}#13, count..][] - * \_LimitExec[1000[INTEGER]] - * \_EvalExec[[scalerank{f}#8 AS rank]] - * \_FieldExtractExec[scalerank{f}#8][] - * \_EsQueryExec[airports], indexMode[standard], query[{" - * esql_single_value":{"field":"scalerank","next":{"range":{"scalerank":{"lt":4,"boost":1.0}}},"source":"rank < 4@3:9"} - * }][_doc{f}#23], limit[], sort[] estimatedRowSize[304] + * EvalExec[[scalerank{f}#8 AS rank]] + * \_LimitExec[1000[INTEGER]] + * \_ExchangeExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8, + * type{f}#9],false + * ] + * \_ProjectExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8, + * type{f}#9] + * ] + * \_FieldExtractExec[abbrev{f}#6, city{f}#12, city_location{f}#13, count..][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "esql_single_value":{"field":"scalerank","next":{"range":{"scalerank":{"lt":4,"boost":1.0}}},"source":"rank < 4@3:9"} + * ][_doc{f}#23], limit[1000], sort[] estimatedRowSize[304] * */ public void testPushWhereEvalToSource() { @@ -3243,7 +3243,8 @@ public void testPushWhereEvalToSource() { """; var plan = this.physicalPlan(query, airports); - var limit = as(plan, LimitExec.class); + var eval = as(plan, EvalExec.class); + var limit = as(eval.child(), LimitExec.class); var exchange = as(limit.child(), ExchangeExec.class); var fragment = as(exchange.child(), FragmentExec.class); var limit2 = as(fragment.fragment(), Limit.class); @@ -3251,16 +3252,14 @@ public void testPushWhereEvalToSource() { assertThat("filter contains LessThan", filter.condition(), instanceOf(LessThan.class)); var optimized = optimizedPlan(plan); - var topLimit = as(optimized, LimitExec.class); + eval = as(optimized, EvalExec.class); + var topLimit = as(eval.child(), LimitExec.class); exchange = as(topLimit.child(), ExchangeExec.class); var project = as(exchange.child(), ProjectExec.class); var fieldExtract = as(project.child(), FieldExtractExec.class); assertThat(fieldExtract.attributesToExtract().size(), greaterThan(5)); - limit = as(fieldExtract.child(), LimitExec.class); - var eval = as(limit.child(), EvalExec.class); - fieldExtract = as(eval.child(), FieldExtractExec.class); - assertThat(fieldExtract.attributesToExtract().stream().map(Attribute::name).collect(Collectors.toList()), contains("scalerank")); var source = source(fieldExtract.child()); + assertThat(source.limit(), is(topLimit.limit())); var condition = as(source.query(), SingleValueQuery.Builder.class); assertThat("Expected predicate to be passed to Lucene query", condition.source().text(), equalTo("rank < 4")); assertThat("Expected field to be passed to Lucene query", condition.field(), equalTo("scalerank")); @@ -3281,7 +3280,8 @@ public void testPushSpatialIntersectsEvalToSource() { """ }) { var plan = this.physicalPlan(query, airports); - var limit = as(plan, LimitExec.class); + var eval = as(plan, EvalExec.class); + var limit = as(eval.child(), LimitExec.class); var exchange = as(limit.child(), ExchangeExec.class); var fragment = as(exchange.child(), FragmentExec.class); var limit2 = as(fragment.fragment(), Limit.class); @@ -3289,16 +3289,14 @@ public void testPushSpatialIntersectsEvalToSource() { assertThat("filter contains ST_INTERSECTS", filter.condition(), instanceOf(SpatialIntersects.class)); var optimized = optimizedPlan(plan); - var topLimit = as(optimized, LimitExec.class); + eval = as(optimized, EvalExec.class); + var topLimit = as(eval.child(), LimitExec.class); exchange = as(topLimit.child(), ExchangeExec.class); var project = as(exchange.child(), ProjectExec.class); var fieldExtract = as(project.child(), FieldExtractExec.class); assertThat(fieldExtract.attributesToExtract().size(), greaterThan(5)); - limit = as(fieldExtract.child(), LimitExec.class); - var eval = as(limit.child(), EvalExec.class); - fieldExtract = as(eval.child(), FieldExtractExec.class); - assertThat(fieldExtract.attributesToExtract().stream().map(Attribute::name).collect(Collectors.toList()), contains("location")); var source = source(fieldExtract.child()); + assertThat(source.limit(), is(topLimit.limit())); var condition = as(source.query(), SpatialRelatesQuery.ShapeQueryBuilder.class); assertThat("Geometry field name", condition.fieldName(), equalTo("location")); assertThat("Spatial relationship", condition.relation(), equalTo(ShapeRelation.INTERSECTS)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java index 80a2d49d0d94a..67b4dd71260aa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java @@ -208,7 +208,7 @@ public void testParenthesizedExpression() { } public void testCommandNamesAsIdentifiers() { - Expression expr = whereExpression("from and where"); + Expression expr = whereExpression("from and limit"); assertThat(expr, instanceOf(And.class)); And and = (And) expr; @@ -216,7 +216,7 @@ public void testCommandNamesAsIdentifiers() { assertThat(((UnresolvedAttribute) and.left()).name(), equalTo("from")); assertThat(and.right(), instanceOf(UnresolvedAttribute.class)); - assertThat(((UnresolvedAttribute) and.right()).name(), equalTo("where")); + assertThat(((UnresolvedAttribute) and.right()).name(), equalTo("limit")); } public void testIdentifiersCaseSensitive() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 53621a79aedac..c797f426d2ae5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -20,14 +20,18 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; +import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; @@ -321,6 +325,61 @@ public void testAggsWithGroupKeyAsAgg() throws Exception { } } + public void testStatsWithGroupKeyAndAggFilter() throws Exception { + var a = attribute("a"); + var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); + var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1)))); + assertEquals( + new Aggregate(EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, List.of(a), List.of(filter, a)), + processingCommand("stats min(a) where a > 1 by a") + ); + } + + public void testStatsWithGroupKeyAndMixedAggAndFilter() throws Exception { + var a = attribute("a"); + var min = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); + var max = new UnresolvedFunction(EMPTY, "max", DEFAULT, List.of(a)); + var avg = new UnresolvedFunction(EMPTY, "avg", DEFAULT, List.of(a)); + var min_alias = new Alias(EMPTY, "min", min); + + var max_filter_ex = new Or( + EMPTY, + new GreaterThan(EMPTY, new Mod(EMPTY, a, integer(3)), integer(10)), + new GreaterThan(EMPTY, new Div(EMPTY, a, integer(2)), integer(100)) + ); + var max_filter = new Alias(EMPTY, "max", new FilteredExpression(EMPTY, max, max_filter_ex)); + + var avg_filter_ex = new GreaterThan(EMPTY, new Div(EMPTY, a, integer(2)), integer(100)); + var avg_filter = new Alias(EMPTY, "avg", new FilteredExpression(EMPTY, avg, avg_filter_ex)); + + assertEquals( + new Aggregate( + EMPTY, + PROCESSING_CMD_INPUT, + Aggregate.AggregateType.STANDARD, + List.of(a), + List.of(min_alias, max_filter, avg_filter, a) + ), + processingCommand(""" + stats + min = min(a), + max = max(a) WHERE (a % 3 > 10 OR a / 2 > 100), + avg = avg(a) WHERE a / 2 > 100 + BY a + """) + ); + } + + public void testStatsWithoutGroupKeyMixedAggAndFilter() throws Exception { + var a = attribute("a"); + var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); + var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1)))); + assertEquals( + new Aggregate(EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, List.of(), List.of(filter)), + processingCommand("stats min(a) where a > 1") + ); + } + public void testInlineStatsWithGroups() { var query = "inlinestats b = min(a) by c, d.e"; if (Build.current().isSnapshot() == false) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index d186b4c199d77..7075c9fe58d63 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -21,6 +21,8 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttributeTests; import org.elasticsearch.xpack.esql.core.expression.UnresolvedNamedExpression; @@ -164,6 +166,16 @@ public void testInfoParameters() throws Exception { * in the parameters and not included. */ expectedCount -= 1; + + // special exceptions with private constructors + if (MetadataAttribute.class.equals(subclass) || ReferenceAttribute.class.equals(subclass)) { + expectedCount++; + } + + if (FieldAttribute.class.equals(subclass)) { + expectedCount += 2; + } + assertEquals(expectedCount, info(node).properties().size()); } @@ -174,6 +186,9 @@ public void testInfoParameters() throws Exception { * implementations in the process. */ public void testTransform() throws Exception { + if (FieldAttribute.class.equals(subclass)) { + assumeTrue("FieldAttribute private constructor", false); + } Constructor ctor = longestCtor(subclass); Object[] nodeCtorArgs = ctorArgs(ctor); T node = ctor.newInstance(nodeCtorArgs); diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java index 081061fd9ceac..369752ea5ed75 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java @@ -46,7 +46,6 @@ public final class RestFreezeIndexAction extends BaseRestHandler { @Override public List routes() { return List.of( - Route.builder(POST, "/{index}/_freeze").deprecated(FREEZE_REMOVED, RestApiVersion.V_7).build(), // Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build() Route.builder(POST, "/{index}/_unfreeze").deprecateAndKeep(UNFREEZE_DEPRECATED).build() ); diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java index 472b6f8087a56..983e972248945 100644 --- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java +++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java @@ -74,14 +74,6 @@ public List routes() { .build(), Route.builder(POST, "/{index}/_graph/explore") .replaces(POST, "/{index}" + URI_BASE + "/graph/_explore", RestApiVersion.V_7) - .build(), - Route.builder(GET, "/{index}/{type}/_graph/explore").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/{type}" + URI_BASE + "/graph/_explore") - .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) - .build(), - Route.builder(POST, "/{index}/{type}/_graph/explore").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}" + URI_BASE + "/graph/_explore") - .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) .build() ); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java index ec8f7c230b1d3..9b7262e8f9b32 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java @@ -34,6 +34,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createFullPolicy; @@ -307,14 +308,16 @@ public void testStepInfoPreservedOnAutoRetry() throws Exception { assertBusy(() -> { Map explainIndex = explainIndex(client(), indexName); - assertThat(explainIndex.get("failed_step_retry_count"), notNullValue()); - assertThat(explainIndex.get("previous_step_info"), notNullValue()); - assertThat((int) explainIndex.get("failed_step_retry_count"), greaterThan(0)); + var assertionMessage = "Assertion failed for the following response: " + explainIndex; + assertThat(assertionMessage, explainIndex.get("failed_step_retry_count"), notNullValue()); + assertThat(assertionMessage, explainIndex.get("previous_step_info"), notNullValue()); + assertThat(assertionMessage, (int) explainIndex.get("failed_step_retry_count"), greaterThan(0)); assertThat( + assertionMessage, explainIndex.get("previous_step_info").toString(), containsString("rollover_alias [" + aliasName + "] does not point to index [" + indexName + "]") ); - }); + }, 30, TimeUnit.SECONDS); } private void assertUnmanagedIndex(Map explainIndexMap) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java index b3f29535020bf..2499cd92113c2 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java @@ -137,7 +137,8 @@ static ClusterState moveClusterStateToStep( lifecycleState, newStepKey, nowSupplier, - forcePhaseDefinitionRefresh + forcePhaseDefinitionRefresh, + true ); return LifecycleExecutionStateUtils.newClusterStateWithLifecycleState(state, idxMeta.getIndex(), newLifecycleState); @@ -175,6 +176,7 @@ static ClusterState moveClusterStateToErrorStep( currentState, new Step.StepKey(currentStep.phase(), currentStep.action(), ErrorStep.NAME), nowSupplier, + false, false ); @@ -243,7 +245,8 @@ static ClusterState moveClusterStateToPreviouslyFailedStep( lifecycleState, nextStepKey, nowSupplier, - forcePhaseDefinitionRefresh + forcePhaseDefinitionRefresh, + false ); LifecycleExecutionState.Builder retryStepState = LifecycleExecutionState.builder(nextStepState); @@ -277,7 +280,8 @@ private static LifecycleExecutionState updateExecutionStateToStep( LifecycleExecutionState existingState, Step.StepKey newStep, LongSupplier nowSupplier, - boolean forcePhaseDefinitionRefresh + boolean forcePhaseDefinitionRefresh, + boolean allowNullPreviousStepInfo ) { Step.StepKey currentStep = Step.getCurrentStepKey(existingState); long nowAsMillis = nowSupplier.getAsLong(); @@ -289,7 +293,9 @@ private static LifecycleExecutionState updateExecutionStateToStep( // clear any step info or error-related settings from the current step updatedState.setFailedStep(null); - updatedState.setPreviousStepInfo(existingState.stepInfo()); + if (allowNullPreviousStepInfo || existingState.stepInfo() != null) { + updatedState.setPreviousStepInfo(existingState.stepInfo()); + } updatedState.setStepInfo(null); updatedState.setIsAutoRetryableError(null); updatedState.setFailedStepRetryCount(null); @@ -390,7 +396,7 @@ public static LifecycleExecutionState moveStateToNextActionAndUpdateCachedPhase( updatedState.setStep(nextStep.name()); updatedState.setStepTime(nowAsMillis); updatedState.setFailedStep(null); - updatedState.setPreviousStepInfo(existingState.stepInfo()); + updatedState.setPreviousStepInfo(null); updatedState.setStepInfo(null); updatedState.setIsAutoRetryableError(null); updatedState.setFailedStepRetryCount(null); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java similarity index 57% rename from x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java rename to x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java index 5d84aad4b7344..083bad2c91613 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java @@ -22,13 +22,13 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.oneOf; -public class DefaultElserIT extends InferenceBaseRestTest { +public class DefaultEndPointsIT extends InferenceBaseRestTest { private TestThreadPool threadPool; @Before public void createThreadPool() { - threadPool = new TestThreadPool(DefaultElserIT.class.getSimpleName()); + threadPool = new TestThreadPool(DefaultEndPointsIT.class.getSimpleName()); } @After @@ -38,7 +38,7 @@ public void tearDown() throws Exception { } @SuppressWarnings("unchecked") - public void testInferCreatesDefaultElser() throws IOException { + public void testInferDeploysDefaultElser() throws IOException { assumeTrue("Default config requires a feature flag", DefaultElserFeatureFlag.isEnabled()); var model = getModel(ElasticsearchInternalService.DEFAULT_ELSER_ID); assertDefaultElserConfig(model); @@ -67,4 +67,39 @@ private static void assertDefaultElserConfig(Map modelConfig) { Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 1, "max_number_of_allocations", 8)) ); } + + @SuppressWarnings("unchecked") + public void testInferDeploysDefaultE5() throws IOException { + assumeTrue("Default config requires a feature flag", DefaultElserFeatureFlag.isEnabled()); + var model = getModel(ElasticsearchInternalService.DEFAULT_E5_ID); + assertDefaultE5Config(model); + + var inputs = List.of("Hello World", "Goodnight moon"); + var queryParams = Map.of("timeout", "120s"); + var results = infer(ElasticsearchInternalService.DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, inputs, queryParams); + var embeddings = (List>) results.get("text_embedding"); + assertThat(results.toString(), embeddings, hasSize(2)); + } + + @SuppressWarnings("unchecked") + private static void assertDefaultE5Config(Map modelConfig) { + assertEquals(modelConfig.toString(), ElasticsearchInternalService.DEFAULT_E5_ID, modelConfig.get("inference_id")); + assertEquals(modelConfig.toString(), ElasticsearchInternalService.NAME, modelConfig.get("service")); + assertEquals(modelConfig.toString(), TaskType.TEXT_EMBEDDING.toString(), modelConfig.get("task_type")); + + var serviceSettings = (Map) modelConfig.get("service_settings"); + assertThat( + modelConfig.toString(), + serviceSettings.get("model_id"), + is(oneOf(".multilingual-e5-small", ".multilingual-e5-small_linux-x86_64")) + ); + assertEquals(modelConfig.toString(), 1, serviceSettings.get("num_threads")); + + var adaptiveAllocations = (Map) serviceSettings.get("adaptive_allocations"); + assertThat( + modelConfig.toString(), + adaptiveAllocations, + Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 1, "max_number_of_allocations", 8)) + ); + } } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 98c8d43707219..cbc50c361e3b5 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -40,7 +40,7 @@ public void testCRUD() throws IOException { } var getAllModels = getAllModels(); - int numModels = DefaultElserFeatureFlag.isEnabled() ? 10 : 9; + int numModels = DefaultElserFeatureFlag.isEnabled() ? 11 : 9; assertThat(getAllModels, hasSize(numModels)); var getSparseModels = getModels("_all", TaskType.SPARSE_EMBEDDING); @@ -51,7 +51,8 @@ public void testCRUD() throws IOException { } var getDenseModels = getModels("_all", TaskType.TEXT_EMBEDDING); - assertThat(getDenseModels, hasSize(4)); + int numDenseModels = DefaultElserFeatureFlag.isEnabled() ? 5 : 4; + assertThat(getDenseModels, hasSize(numDenseModels)); for (var denseModel : getDenseModels) { assertEquals("text_embedding", denseModel.get("task_type")); } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index a76c4303268e4..e62cdcdc7fd2a 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -11,7 +11,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; @@ -47,6 +50,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.equalTo; @@ -57,6 +61,8 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; public class ModelRegistryIT extends ESSingleNodeTestCase { @@ -122,7 +128,12 @@ public void testGetModel() throws Exception { assertEquals(model.getConfigurations().getService(), modelHolder.get().service()); var elserService = new ElasticsearchInternalService( - new InferenceServiceExtension.InferenceServiceFactoryContext(mock(Client.class), mock(ThreadPool.class)) + new InferenceServiceExtension.InferenceServiceFactoryContext( + mock(Client.class), + mock(ThreadPool.class), + mock(ClusterService.class), + Settings.EMPTY + ) ); ElasticsearchInternalModel roundTripModel = (ElasticsearchInternalModel) elserService.parsePersistedConfigWithSecrets( modelHolder.get().inferenceEntityId(), @@ -283,18 +294,30 @@ public void testGetModelWithSecrets() throws InterruptedException { } public void testGetAllModels_WithDefaults() throws Exception { - var service = "foo"; - var secret = "abc"; + var serviceName = "foo"; int configuredModelCount = 10; int defaultModelCount = 2; int totalModelCount = 12; - var defaultConfigs = new HashMap(); + var service = mock(InferenceService.class); + + var defaultConfigs = new ArrayList(); + var defaultIds = new ArrayList(); for (int i = 0; i < defaultModelCount; i++) { var id = "default-" + i; - defaultConfigs.put(id, createUnparsedConfig(id, randomFrom(TaskType.values()), service, secret)); + var taskType = randomFrom(TaskType.values()); + defaultConfigs.add(createModel(id, taskType, serviceName)); + defaultIds.add(new InferenceService.DefaultConfigId(id, taskType, service)); } - defaultConfigs.values().forEach(modelRegistry::addDefaultConfiguration); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener>) invocation.getArguments()[0]; + listener.onResponse(defaultConfigs); + return Void.TYPE; + }).when(service).defaultConfigs(any()); + + defaultIds.forEach(modelRegistry::addDefaultIds); AtomicReference putModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); @@ -302,7 +325,7 @@ public void testGetAllModels_WithDefaults() throws Exception { var createdModels = new HashMap(); for (int i = 0; i < configuredModelCount; i++) { var id = randomAlphaOfLength(5) + i; - var model = createModel(id, randomFrom(TaskType.values()), service); + var model = createModel(id, randomFrom(TaskType.values()), serviceName); createdModels.put(id, model); blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); @@ -316,16 +339,22 @@ public void testGetAllModels_WithDefaults() throws Exception { var getAllModels = modelHolder.get(); assertReturnModelIsModifiable(modelHolder.get().get(0)); + // same result but configs should have been persisted this time + blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(modelHolder.get(), hasSize(totalModelCount)); + // sort in the same order as the returned models - var ids = new ArrayList<>(defaultConfigs.keySet().stream().toList()); + var ids = new ArrayList<>(defaultIds.stream().map(InferenceService.DefaultConfigId::inferenceId).toList()); ids.addAll(createdModels.keySet().stream().toList()); ids.sort(String::compareTo); + var configsById = defaultConfigs.stream().collect(Collectors.toMap(Model::getInferenceEntityId, Function.identity())); for (int i = 0; i < totalModelCount; i++) { var id = ids.get(i); assertEquals(id, getAllModels.get(i).inferenceEntityId()); if (id.startsWith("default")) { - assertEquals(defaultConfigs.get(id).taskType(), getAllModels.get(i).taskType()); - assertEquals(defaultConfigs.get(id).service(), getAllModels.get(i).service()); + assertEquals(configsById.get(id).getTaskType(), getAllModels.get(i).taskType()); + assertEquals(configsById.get(id).getConfigurations().getService(), getAllModels.get(i).service()); } else { assertEquals(createdModels.get(id).getTaskType(), getAllModels.get(i).taskType()); assertEquals(createdModels.get(id).getConfigurations().getService(), getAllModels.get(i).service()); @@ -334,16 +363,27 @@ public void testGetAllModels_WithDefaults() throws Exception { } public void testGetAllModels_OnlyDefaults() throws Exception { - var service = "foo"; - var secret = "abc"; int defaultModelCount = 2; + var serviceName = "foo"; + var service = mock(InferenceService.class); - var defaultConfigs = new HashMap(); + var defaultConfigs = new ArrayList(); + var defaultIds = new ArrayList(); for (int i = 0; i < defaultModelCount; i++) { var id = "default-" + i; - defaultConfigs.put(id, createUnparsedConfig(id, randomFrom(TaskType.values()), service, secret)); + var taskType = randomFrom(TaskType.values()); + defaultConfigs.add(createModel(id, taskType, serviceName)); + defaultIds.add(new InferenceService.DefaultConfigId(id, taskType, service)); } - defaultConfigs.values().forEach(modelRegistry::addDefaultConfiguration); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener>) invocation.getArguments()[0]; + listener.onResponse(defaultConfigs); + return Void.TYPE; + }).when(service).defaultConfigs(any()); + + defaultIds.forEach(modelRegistry::addDefaultIds); AtomicReference exceptionHolder = new AtomicReference<>(); AtomicReference> modelHolder = new AtomicReference<>(); @@ -354,31 +394,42 @@ public void testGetAllModels_OnlyDefaults() throws Exception { assertReturnModelIsModifiable(modelHolder.get().get(0)); // sort in the same order as the returned models - var ids = new ArrayList<>(defaultConfigs.keySet().stream().toList()); + var configsById = defaultConfigs.stream().collect(Collectors.toMap(Model::getInferenceEntityId, Function.identity())); + var ids = new ArrayList<>(configsById.keySet().stream().toList()); ids.sort(String::compareTo); for (int i = 0; i < defaultModelCount; i++) { var id = ids.get(i); assertEquals(id, getAllModels.get(i).inferenceEntityId()); - assertEquals(defaultConfigs.get(id).taskType(), getAllModels.get(i).taskType()); - assertEquals(defaultConfigs.get(id).service(), getAllModels.get(i).service()); + assertEquals(configsById.get(id).getTaskType(), getAllModels.get(i).taskType()); + assertEquals(configsById.get(id).getConfigurations().getService(), getAllModels.get(i).service()); } } public void testGet_WithDefaults() throws InterruptedException { - var service = "foo"; - var secret = "abc"; + var serviceName = "foo"; + var service = mock(InferenceService.class); + + var defaultConfigs = new ArrayList(); + var defaultIds = new ArrayList(); - var defaultSparse = createUnparsedConfig("default-sparse", TaskType.SPARSE_EMBEDDING, service, secret); - var defaultText = createUnparsedConfig("default-text", TaskType.TEXT_EMBEDDING, service, secret); + defaultConfigs.add(createModel("default-sparse", TaskType.SPARSE_EMBEDDING, serviceName)); + defaultConfigs.add(createModel("default-text", TaskType.TEXT_EMBEDDING, serviceName)); + defaultIds.add(new InferenceService.DefaultConfigId("default-sparse", TaskType.SPARSE_EMBEDDING, service)); + defaultIds.add(new InferenceService.DefaultConfigId("default-text", TaskType.TEXT_EMBEDDING, service)); - modelRegistry.addDefaultConfiguration(defaultSparse); - modelRegistry.addDefaultConfiguration(defaultText); + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener>) invocation.getArguments()[0]; + listener.onResponse(defaultConfigs); + return Void.TYPE; + }).when(service).defaultConfigs(any()); + defaultIds.forEach(modelRegistry::addDefaultIds); AtomicReference putModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - var configured1 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), service); - var configured2 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), service); + var configured1 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), serviceName); + var configured2 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), serviceName); blockingCall(listener -> modelRegistry.storeModel(configured1, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); blockingCall(listener -> modelRegistry.storeModel(configured2, listener), putModelHolder, exceptionHolder); @@ -387,6 +438,7 @@ public void testGet_WithDefaults() throws InterruptedException { AtomicReference modelHolder = new AtomicReference<>(); blockingCall(listener -> modelRegistry.getModel("default-sparse", listener), modelHolder, exceptionHolder); + assertNull(exceptionHolder.get()); assertEquals("default-sparse", modelHolder.get().inferenceEntityId()); assertEquals(TaskType.SPARSE_EMBEDDING, modelHolder.get().taskType()); assertReturnModelIsModifiable(modelHolder.get()); @@ -401,23 +453,32 @@ public void testGet_WithDefaults() throws InterruptedException { } public void testGetByTaskType_WithDefaults() throws Exception { - var service = "foo"; - var secret = "abc"; - - var defaultSparse = createUnparsedConfig("default-sparse", TaskType.SPARSE_EMBEDDING, service, secret); - var defaultText = createUnparsedConfig("default-text", TaskType.TEXT_EMBEDDING, service, secret); - var defaultChat = createUnparsedConfig("default-chat", TaskType.COMPLETION, service, secret); - - modelRegistry.addDefaultConfiguration(defaultSparse); - modelRegistry.addDefaultConfiguration(defaultText); - modelRegistry.addDefaultConfiguration(defaultChat); + var serviceName = "foo"; + + var defaultSparse = createModel("default-sparse", TaskType.SPARSE_EMBEDDING, serviceName); + var defaultText = createModel("default-text", TaskType.TEXT_EMBEDDING, serviceName); + var defaultChat = createModel("default-chat", TaskType.COMPLETION, serviceName); + + var service = mock(InferenceService.class); + var defaultIds = new ArrayList(); + defaultIds.add(new InferenceService.DefaultConfigId("default-sparse", TaskType.SPARSE_EMBEDDING, service)); + defaultIds.add(new InferenceService.DefaultConfigId("default-text", TaskType.TEXT_EMBEDDING, service)); + defaultIds.add(new InferenceService.DefaultConfigId("default-chat", TaskType.COMPLETION, service)); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener>) invocation.getArguments()[0]; + listener.onResponse(List.of(defaultSparse, defaultChat, defaultText)); + return Void.TYPE; + }).when(service).defaultConfigs(any()); + defaultIds.forEach(modelRegistry::addDefaultIds); AtomicReference putModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - var configuredSparse = createModel("configured-sparse", TaskType.SPARSE_EMBEDDING, service); - var configuredText = createModel("configured-text", TaskType.TEXT_EMBEDDING, service); - var configuredRerank = createModel("configured-rerank", TaskType.RERANK, service); + var configuredSparse = createModel("configured-sparse", TaskType.SPARSE_EMBEDDING, serviceName); + var configuredText = createModel("configured-text", TaskType.TEXT_EMBEDDING, serviceName); + var configuredRerank = createModel("configured-rerank", TaskType.RERANK, serviceName); blockingCall(listener -> modelRegistry.storeModel(configuredSparse, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); blockingCall(listener -> modelRegistry.storeModel(configuredText, listener), putModelHolder, exceptionHolder); @@ -531,10 +592,6 @@ public static Model createModelWithSecrets(String inferenceEntityId, TaskType ta ); } - public static UnparsedModel createUnparsedConfig(String inferenceEntityId, TaskType taskType, String service, String secret) { - return new UnparsedModel(inferenceEntityId, taskType, service, Map.of("a", "b"), Map.of("secret", secret)); - } - private static class TestModelOfAnyKind extends ModelConfigurations { record TestModelServiceSettings() implements ServiceSettings { diff --git a/x-pack/plugin/inference/src/main/java/module-info.java b/x-pack/plugin/inference/src/main/java/module-info.java index 53cb6ac154ced..60cb254e0afbe 100644 --- a/x-pack/plugin/inference/src/main/java/module-info.java +++ b/x-pack/plugin/inference/src/main/java/module-info.java @@ -32,6 +32,7 @@ requires software.amazon.awssdk.profiles; requires org.slf4j; requires software.amazon.awssdk.retries.api; + requires org.reactivestreams; exports org.elasticsearch.xpack.inference.action; exports org.elasticsearch.xpack.inference.registry; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index d251120980e0b..ebbf1e59e8b1f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -212,13 +212,21 @@ public Collection createComponents(PluginServices services) { ); } - var factoryContext = new InferenceServiceExtension.InferenceServiceFactoryContext(services.client(), services.threadPool()); + var factoryContext = new InferenceServiceExtension.InferenceServiceFactoryContext( + services.client(), + services.threadPool(), + services.clusterService(), + settings + ); + // This must be done after the HttpRequestSenderFactory is created so that the services can get the // reference correctly var registry = new InferenceServiceRegistry(inferenceServices, factoryContext); registry.init(services.client()); - for (var service : registry.getServices().values()) { - service.defaultConfigs().forEach(modelRegistry::addDefaultConfiguration); + if (DefaultElserFeatureFlag.isEnabled()) { + for (var service : registry.getServices().values()) { + service.defaultConfigIds().forEach(modelRegistry::addDefaultIds); + } } inferenceServiceRegistry.set(registry); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java index 5ee1e40869dbc..55aad5c55a2ac 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java @@ -9,13 +9,13 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.inference.InferenceServiceRegistry; -import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; @@ -29,8 +29,11 @@ import org.elasticsearch.xpack.inference.registry.ModelRegistry; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.concurrent.Executor; +import java.util.stream.Collectors; public class TransportGetInferenceModelAction extends HandledTransportAction< GetInferenceModelAction.Request, @@ -96,38 +99,69 @@ private void getSingleModel( var model = service.get() .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings()); - delegate.onResponse(new GetInferenceModelAction.Response(List.of(model.getConfigurations()))); + + service.get() + .updateModelsWithDynamicFields( + List.of(model), + delegate.delegateFailureAndWrap( + (l2, updatedModels) -> l2.onResponse( + new GetInferenceModelAction.Response( + updatedModels.stream().map(Model::getConfigurations).collect(Collectors.toList()) + ) + ) + ) + ); })); } private void getAllModels(ActionListener listener) { - modelRegistry.getAllModels( - listener.delegateFailureAndWrap((l, models) -> executor.execute(ActionRunnable.supply(l, () -> parseModels(models)))) - ); + modelRegistry.getAllModels(listener.delegateFailureAndWrap((l, models) -> executor.execute(() -> parseModels(models, listener)))); } private void getModelsByTaskType(TaskType taskType, ActionListener listener) { modelRegistry.getModelsByTaskType( taskType, - listener.delegateFailureAndWrap((l, models) -> executor.execute(ActionRunnable.supply(l, () -> parseModels(models)))) + listener.delegateFailureAndWrap((l, models) -> executor.execute(() -> parseModels(models, listener))) ); } - private GetInferenceModelAction.Response parseModels(List unparsedModels) { - var parsedModels = new ArrayList(); - - for (var unparsedModel : unparsedModels) { - var service = serviceRegistry.getService(unparsedModel.service()); - if (service.isEmpty()) { - throw serviceNotFoundException(unparsedModel.service(), unparsedModel.inferenceEntityId()); + private void parseModels(List unparsedModels, ActionListener listener) { + var parsedModelsByService = new HashMap>(); + try { + for (var unparsedModel : unparsedModels) { + var service = serviceRegistry.getService(unparsedModel.service()); + if (service.isEmpty()) { + throw serviceNotFoundException(unparsedModel.service(), unparsedModel.inferenceEntityId()); + } + var list = parsedModelsByService.computeIfAbsent(service.get().name(), s -> new ArrayList<>()); + list.add( + service.get() + .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings()) + ); } - parsedModels.add( - service.get() - .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings()) - .getConfigurations() + + var groupedListener = new GroupedActionListener>( + parsedModelsByService.entrySet().size(), + listener.delegateFailureAndWrap((delegate, listOfListOfModels) -> { + var modifiable = new ArrayList(); + for (var l : listOfListOfModels) { + modifiable.addAll(l); + } + modifiable.sort(Comparator.comparing(Model::getInferenceEntityId)); + delegate.onResponse( + new GetInferenceModelAction.Response(modifiable.stream().map(Model::getConfigurations).collect(Collectors.toList())) + ); + }) ); + + for (var entry : parsedModelsByService.entrySet()) { + serviceRegistry.getService(entry.getKey()) + .get() // must be non-null to get this far + .updateModelsWithDynamicFields(entry.getValue(), groupedListener); + } + } catch (Exception e) { + listener.onFailure(e); } - return new GetInferenceModelAction.Response(parsedModels); } private ElasticsearchStatusException serviceNotFoundException(String service, String inferenceId) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java index 477c3ea6352f5..20520ca829297 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java @@ -13,7 +13,7 @@ import java.util.Map; public class ChunkingSettingsBuilder { - public static final WordBoundaryChunkingSettings DEFAULT_SETTINGS = new WordBoundaryChunkingSettings(250, 100); + public static final SentenceBoundaryChunkingSettings DEFAULT_SETTINGS = new SentenceBoundaryChunkingSettings(250, 1); public static ChunkingSettings fromMap(Map settings) { if (settings.isEmpty()) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java index 758dd5d04e268..04a07eeb984ec 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java @@ -35,7 +35,7 @@ public class SentenceBoundaryChunkingSettings implements ChunkingSettings { ChunkingSettingsOptions.SENTENCE_OVERLAP.toString() ); - private static int DEFAULT_OVERLAP = 0; + private static int DEFAULT_OVERLAP = 1; protected final int maxChunkSize; protected int sentenceOverlap = DEFAULT_OVERLAP; @@ -69,17 +69,18 @@ public static SentenceBoundaryChunkingSettings fromMap(Map map) validationException ); - Integer sentenceOverlap = ServiceUtils.extractOptionalPositiveInteger( + Integer sentenceOverlap = ServiceUtils.removeAsType( map, ChunkingSettingsOptions.SENTENCE_OVERLAP.toString(), - ModelConfigurations.CHUNKING_SETTINGS, + Integer.class, validationException ); - - if (sentenceOverlap != null && sentenceOverlap > 1) { + if (sentenceOverlap == null) { + sentenceOverlap = DEFAULT_OVERLAP; + } else if (sentenceOverlap > 1 || sentenceOverlap < 0) { validationException.addValidationError( - ChunkingSettingsOptions.SENTENCE_OVERLAP.toString() + "[" + sentenceOverlap + "] must be either 0 or 1" - ); // todo better + ChunkingSettingsOptions.SENTENCE_OVERLAP + "[" + sentenceOverlap + "] must be either 0 or 1" + ); } if (validationException.validationErrors().isEmpty() == false) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java deleted file mode 100644 index 3871b5fb98882..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.googleaistudio; - -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; -import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; -import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel; - -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; - -public class GoogleAiStudioActionCreator implements GoogleAiStudioActionVisitor { - - private static final String COMPLETION_ERROR_MESSAGE = "Google AI Studio completion"; - private final Sender sender; - - private final ServiceComponents serviceComponents; - - public GoogleAiStudioActionCreator(Sender sender, ServiceComponents serviceComponents) { - this.sender = Objects.requireNonNull(sender); - this.serviceComponents = Objects.requireNonNull(serviceComponents); - } - - @Override - public ExecutableAction create(GoogleAiStudioCompletionModel model, Map taskSettings) { - // no overridden model as task settings are always empty for Google AI Studio completion model - var requestManager = new GoogleAiStudioCompletionRequestManager(model, serviceComponents.threadPool()); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), COMPLETION_ERROR_MESSAGE); - return new SingleInputSenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage, COMPLETION_ERROR_MESSAGE); - } - - @Override - public ExecutableAction create(GoogleAiStudioEmbeddingsModel model, Map taskSettings) { - var requestManager = new GoogleAiStudioEmbeddingsRequestManager( - model, - serviceComponents.truncator(), - serviceComponents.threadPool() - ); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio embeddings"); - return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java deleted file mode 100644 index 2e89200cce53b..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.googleaistudio; - -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; -import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel; - -import java.util.Map; - -public interface GoogleAiStudioActionVisitor { - - ExecutableAction create(GoogleAiStudioCompletionModel model, Map taskSettings); - - ExecutableAction create(GoogleAiStudioEmbeddingsModel model, Map taskSettings); -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java index a4e0c399517c1..2afa91d4dc776 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockChatCompletionExecutor.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockChatCompletionRequest; import org.elasticsearch.xpack.inference.external.response.amazonbedrock.AmazonBedrockResponseHandler; import org.elasticsearch.xpack.inference.external.response.amazonbedrock.completion.AmazonBedrockChatCompletionResponseListener; @@ -33,11 +34,16 @@ protected AmazonBedrockChatCompletionExecutor( @Override protected void executeClientRequest(AmazonBedrockBaseClient awsBedrockClient) { - var chatCompletionResponseListener = new AmazonBedrockChatCompletionResponseListener( - chatCompletionRequest, - responseHandler, - inferenceResultsListener - ); - chatCompletionRequest.executeChatCompletionRequest(awsBedrockClient, chatCompletionResponseListener); + if (chatCompletionRequest.isStreaming()) { + var publisher = chatCompletionRequest.executeStreamChatCompletionRequest(awsBedrockClient); + inferenceResultsListener.onResponse(new StreamingChatCompletionResults(publisher)); + } else { + var chatCompletionResponseListener = new AmazonBedrockChatCompletionResponseListener( + chatCompletionRequest, + responseHandler, + inferenceResultsListener + ); + chatCompletionRequest.executeChatCompletionRequest(awsBedrockClient, chatCompletionResponseListener); + } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java index 23b6884ddc33a..f1cfc84643b1c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockClient.java @@ -9,17 +9,22 @@ import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; import software.amazon.awssdk.services.bedrockruntime.model.ConverseResponse; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest; import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelRequest; import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import java.time.Instant; +import java.util.concurrent.Flow; public interface AmazonBedrockClient { void converse(ConverseRequest converseRequest, ActionListener responseListener) throws ElasticsearchException; + Flow.Publisher converseStream(ConverseStreamRequest converseStreamRequest) throws ElasticsearchException; + void invokeModel(InvokeModelRequest invokeModelRequest, ActionListener responseListener) throws ElasticsearchException; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java index b1486f4995b84..040aa99d81346 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClient.java @@ -17,16 +17,21 @@ import software.amazon.awssdk.services.bedrockruntime.model.BedrockRuntimeException; import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; import software.amazon.awssdk.services.bedrockruntime.model.ConverseResponse; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler; import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelRequest; import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.SpecialPermission; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockModel; +import org.reactivestreams.FlowAdapters; import org.slf4j.LoggerFactory; import java.security.AccessController; @@ -36,6 +41,7 @@ import java.util.Objects; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Flow; /** * Not marking this as "final" so we can subclass it for mocking @@ -53,19 +59,21 @@ public class AmazonBedrockInferenceClient extends AmazonBedrockBaseClient { private static final Duration DEFAULT_CLIENT_TIMEOUT_MS = Duration.ofMillis(10000); private final BedrockRuntimeAsyncClient internalClient; + private final ThreadPool threadPool; private volatile Instant expiryTimestamp; - public static AmazonBedrockBaseClient create(AmazonBedrockModel model, @Nullable TimeValue timeout) { + public static AmazonBedrockBaseClient create(AmazonBedrockModel model, @Nullable TimeValue timeout, ThreadPool threadPool) { try { - return new AmazonBedrockInferenceClient(model, timeout); + return new AmazonBedrockInferenceClient(model, timeout, threadPool); } catch (Exception e) { throw new ElasticsearchException("Failed to create Amazon Bedrock Client", e); } } - protected AmazonBedrockInferenceClient(AmazonBedrockModel model, @Nullable TimeValue timeout) { + protected AmazonBedrockInferenceClient(AmazonBedrockModel model, @Nullable TimeValue timeout, ThreadPool threadPool) { super(model, timeout); this.internalClient = createAmazonBedrockClient(model, timeout); + this.threadPool = Objects.requireNonNull(threadPool); setExpiryTimestamp(); } @@ -79,6 +87,16 @@ public void converse(ConverseRequest converseRequest, ActionListener converseStream(ConverseStreamRequest request) throws ElasticsearchException { + var awsResponseProcessor = new AmazonBedrockStreamingChatProcessor(threadPool); + internalClient.converseStream( + request, + ConverseStreamResponseHandler.builder().subscriber(() -> FlowAdapters.toSubscriber(awsResponseProcessor)).build() + ); + return awsResponseProcessor; + } + private void onFailure(ActionListener listener, Throwable t, String method) { var unwrappedException = t; if (t instanceof CompletionException || t instanceof ExecutionException) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java index 21e5cfaf211e5..339673e1302ac 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCache.java @@ -29,12 +29,9 @@ public final class AmazonBedrockInferenceClientCache implements AmazonBedrockCli // not final for testing private Clock clock; - public AmazonBedrockInferenceClientCache( - BiFunction creator, - @Nullable Clock clock - ) { + public AmazonBedrockInferenceClientCache(BiFunction creator, Clock clock) { this.creator = Objects.requireNonNull(creator); - this.clock = Objects.requireNonNullElse(clock, Clock.systemUTC()); + this.clock = Objects.requireNonNull(clock); } public AmazonBedrockBaseClient getOrCreateClient(AmazonBedrockModel model, @Nullable TimeValue timeout) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java index e23b0274ede26..a8d85d896d684 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSender.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.inference.services.ServiceComponents; import java.io.IOException; +import java.time.Clock; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -42,7 +43,10 @@ public Factory(ServiceComponents serviceComponents, ClusterService clusterServic } public Sender createSender() { - var clientCache = new AmazonBedrockInferenceClientCache(AmazonBedrockInferenceClient::create, null); + var clientCache = new AmazonBedrockInferenceClientCache( + (model, timeout) -> AmazonBedrockInferenceClient.create(model, timeout, serviceComponents.threadPool()), + Clock.systemUTC() + ); return createSender(new AmazonBedrockExecuteOnlyRequestSender(clientCache, serviceComponents.throttlerManager())); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessor.java new file mode 100644 index 0000000000000..439fc5b65efd5 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessor.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.amazonbedrock; + +import software.amazon.awssdk.services.bedrockruntime.model.ContentBlockDeltaEvent; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamOutput; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Strings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; + +import java.util.ArrayDeque; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; + +class AmazonBedrockStreamingChatProcessor implements Flow.Processor { + private final AtomicReference error = new AtomicReference<>(null); + private final AtomicLong demand = new AtomicLong(0); + private final AtomicBoolean isDone = new AtomicBoolean(false); + private final AtomicBoolean onCompleteCalled = new AtomicBoolean(false); + private final AtomicBoolean onErrorCalled = new AtomicBoolean(false); + private final ThreadPool threadPool; + private volatile Flow.Subscriber downstream; + private volatile Flow.Subscription upstream; + + AmazonBedrockStreamingChatProcessor(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + @Override + public void subscribe(Flow.Subscriber subscriber) { + if (downstream == null) { + downstream = subscriber; + downstream.onSubscribe(new StreamSubscription()); + } else { + subscriber.onError(new IllegalStateException("Subscriber already set.")); + } + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (upstream == null) { + upstream = subscription; + var currentRequestCount = demand.getAndUpdate(i -> 0); + if (currentRequestCount > 0) { + upstream.request(currentRequestCount); + } + } else { + subscription.cancel(); + } + } + + @Override + public void onNext(ConverseStreamOutput item) { + if (item.sdkEventType() == ConverseStreamOutput.EventType.CONTENT_BLOCK_DELTA) { + demand.set(0); // reset demand before we fork to another thread + item.accept(ConverseStreamResponseHandler.Visitor.builder().onContentBlockDelta(this::sendDownstreamOnAnotherThread).build()); + } else { + upstream.request(1); + } + } + + // this is always called from a netty thread maintained by the AWS SDK, we'll move it to our thread to process the response + private void sendDownstreamOnAnotherThread(ContentBlockDeltaEvent event) { + CompletableFuture.runAsync(() -> { + var text = event.delta().text(); + var result = new ArrayDeque(1); + result.offer(new StreamingChatCompletionResults.Result(text)); + var results = new StreamingChatCompletionResults.Results(result); + downstream.onNext(results); + }, threadPool.executor(UTILITY_THREAD_POOL_NAME)); + } + + @Override + public void onError(Throwable amazonBedrockRuntimeException) { + error.set( + new ElasticsearchException( + Strings.format("AmazonBedrock StreamingChatProcessor failure: [%s]", amazonBedrockRuntimeException.getMessage()), + amazonBedrockRuntimeException + ) + ); + if (isDone.compareAndSet(false, true) && checkAndResetDemand() && onErrorCalled.compareAndSet(false, true)) { + downstream.onError(error.get()); + } + } + + private boolean checkAndResetDemand() { + return demand.getAndUpdate(i -> 0L) > 0L; + } + + @Override + public void onComplete() { + if (isDone.compareAndSet(false, true) && checkAndResetDemand() && onCompleteCalled.compareAndSet(false, true)) { + downstream.onComplete(); + } + } + + private class StreamSubscription implements Flow.Subscription { + @Override + public void request(long n) { + if (n > 0L) { + demand.updateAndGet(i -> { + var sum = i + n; + return sum >= 0 ? sum : Long.MAX_VALUE; + }); + if (upstream == null) { + // wait for upstream to subscribe before forwarding request + return; + } + if (upstreamIsRunning()) { + requestOnMlThread(n); + } else if (error.get() != null && onErrorCalled.compareAndSet(false, true)) { + downstream.onError(error.get()); + } else if (onCompleteCalled.compareAndSet(false, true)) { + downstream.onComplete(); + } + } else { + cancel(); + downstream.onError(new IllegalStateException("Cannot request a negative number.")); + } + } + + private boolean upstreamIsRunning() { + return isDone.get() == false && error.get() == null; + } + + private void requestOnMlThread(long n) { + var currentThreadPool = EsExecutors.executorName(Thread.currentThread().getName()); + if (UTILITY_THREAD_POOL_NAME.equalsIgnoreCase(currentThreadPool)) { + upstream.request(n); + } else { + CompletableFuture.runAsync(() -> upstream.request(n), threadPool.executor(UTILITY_THREAD_POOL_NAME)); + } + } + + @Override + public void cancel() { + if (upstream != null && upstreamIsRunning()) { + upstream.cancel(); + } + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java index 4ba5b552f802a..0241dcd6142a6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java @@ -8,23 +8,48 @@ package org.elasticsearch.xpack.inference.external.googleaistudio; import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioErrorResponseEntity; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import java.io.IOException; +import java.util.concurrent.Flow; + import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; public class GoogleAiStudioResponseHandler extends BaseResponseHandler { static final String GOOGLE_AI_STUDIO_UNAVAILABLE = "The Google AI Studio service may be temporarily overloaded or down"; + private final boolean canHandleStreamingResponses; + private final CheckedFunction content; public GoogleAiStudioResponseHandler(String requestType, ResponseParser parseFunction) { + this(requestType, parseFunction, false, xContentParser -> { + assert false : "do not call this"; + return ""; + }); + } + + public GoogleAiStudioResponseHandler( + String requestType, + ResponseParser parseFunction, + boolean canHandleStreamingResponses, + CheckedFunction content + ) { super(requestType, parseFunction, GoogleAiStudioErrorResponseEntity::fromResponse); + this.canHandleStreamingResponses = canHandleStreamingResponses; + this.content = content; } @Override @@ -72,4 +97,18 @@ private static String resourceNotFoundError(Request request) { return format("Resource not found at [%s]", request.getURI()); } + @Override + public boolean canHandleStreamingResponses() { + return canHandleStreamingResponses; + } + + @Override + public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { + var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); + var googleAiProcessor = new GoogleAiStudioStreamingProcessor(content); + flow.subscribe(serverSentEventProcessor); + serverSentEventProcessor.subscribe(googleAiProcessor); + return new StreamingChatCompletionResults(googleAiProcessor); + } + } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessor.java new file mode 100644 index 0000000000000..aa1232f4182e3 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessor.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.googleaistudio; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; +import org.elasticsearch.xpack.inference.common.DelegatingProcessor; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; + +class GoogleAiStudioStreamingProcessor extends DelegatingProcessor, StreamingChatCompletionResults.Results> { + private static final Logger log = LogManager.getLogger(GoogleAiStudioStreamingProcessor.class); + private final CheckedFunction content; + + GoogleAiStudioStreamingProcessor(CheckedFunction content) { + this.content = content; + } + + @Override + protected void next(Deque item) throws Exception { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + var results = new ArrayDeque(item.size()); + for (ServerSentEvent event : item) { + if (ServerSentEventField.DATA == event.name() && event.hasValue()) { + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, event.value())) { + var delta = content.apply(jsonParser); + results.offer(new StreamingChatCompletionResults.Result(delta)); + } catch (Exception e) { + log.warn("Failed to parse event from inference provider: {}", event); + throw e; + } + } + } + + if (results.isEmpty()) { + upstream().request(1); + } else { + downstream().onNext(new StreamingChatCompletionResults.Results(results)); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java index 6b04b66cb7c11..f0102d01b37a1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java @@ -153,44 +153,31 @@ public void stream(HttpRequest request, HttpContext context, ActionListener client.execute( - request.requestProducer(), - new StreamingHttpResultPublisher(threadPool, settings, callOnceListener), - context, - new FutureCallback<>() { - @Override - public void completed(HttpResponse response) { - // StreamingHttpResultPublisher will publish results to the Flow.Publisher returned in the ActionListener - } - - @Override - public void failed(Exception ex) { - throttlerManager.warn( - logger, - format("Request from inference entity id [%s] failed", request.inferenceEntityId()), - ex - ); - failUsingUtilityThread(ex, callOnceListener); - } - - @Override - public void cancelled() { - failUsingUtilityThread( + var streamingProcessor = new StreamingHttpResultPublisher(threadPool, settings, listener); + + SocketAccess.doPrivileged(() -> client.execute(request.requestProducer(), streamingProcessor, context, new FutureCallback<>() { + @Override + public void completed(HttpResponse response) { + streamingProcessor.close(); + } + + @Override + public void failed(Exception ex) { + threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> streamingProcessor.failed(ex)); + } + + @Override + public void cancelled() { + threadPool.executor(UTILITY_THREAD_POOL_NAME) + .execute( + () -> streamingProcessor.failed( new CancellationException( format("Request from inference entity id [%s] was cancelled", request.inferenceEntityId()) - ), - callOnceListener - ); - } - } - ) - ); + ) + ) + ); + } + })); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java index 3fd5d02ef4679..bf74ca86a969a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisher.java @@ -65,7 +65,7 @@ class StreamingHttpResultPublisher implements HttpAsyncResponseConsumer> listener) { this.settings = Objects.requireNonNull(settings); - this.listener = Objects.requireNonNull(listener); + this.listener = ActionListener.notifyOnce(Objects.requireNonNull(listener)); this.taskRunner = new RequestBasedTaskRunner(new OffloadThread(), threadPool, UTILITY_THREAD_POOL_NAME); } @@ -152,9 +152,13 @@ public void responseCompleted(HttpContext httpContext) {} @Override public void failed(Exception e) { if (this.isDone.compareAndSet(false, true)) { - ex = e; - queue.offer(() -> subscriber.onError(e)); - taskRunner.requestNextRun(); + if (listenerCalled.compareAndSet(false, true)) { + listener.onFailure(e); + } else { + ex = e; + queue.offer(() -> subscriber.onError(e)); + taskRunner.requestNextRun(); + } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java index 1c6bb58717942..69a5c665feb86 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java @@ -22,7 +22,6 @@ import org.elasticsearch.xpack.inference.external.response.amazonbedrock.completion.AmazonBedrockChatCompletionResponseHandler; import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionModel; -import java.util.List; import java.util.function.Supplier; public class AmazonBedrockChatCompletionRequestManager extends AmazonBedrockRequestManager { @@ -45,9 +44,11 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); + var docsOnly = DocumentsOnlyInput.of(inferenceInputs); + var docsInput = docsOnly.getInputs(); + var stream = docsOnly.stream(); var requestEntity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, docsInput); - var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, timeout); + var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, timeout, stream); var responseHandler = new AmazonBedrockChatCompletionResponseHandler(); try { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java index 426102f7f2376..abe50c6fae3f9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioCompletionResponseEntity; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; -import java.util.List; import java.util.Objects; import java.util.function.Supplier; @@ -32,7 +31,12 @@ public class GoogleAiStudioCompletionRequestManager extends GoogleAiStudioReques private final GoogleAiStudioCompletionModel model; private static ResponseHandler createCompletionHandler() { - return new GoogleAiStudioResponseHandler("google ai studio completion", GoogleAiStudioCompletionResponseEntity::fromResponse); + return new GoogleAiStudioResponseHandler( + "google ai studio completion", + GoogleAiStudioCompletionResponseEntity::fromResponse, + true, + GoogleAiStudioCompletionResponseEntity::content + ); } public GoogleAiStudioCompletionRequestManager(GoogleAiStudioCompletionModel model, ThreadPool threadPool) { @@ -47,8 +51,7 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); - GoogleAiStudioCompletionRequest request = new GoogleAiStudioCompletionRequest(docsInput, model); + GoogleAiStudioCompletionRequest request = new GoogleAiStudioCompletionRequest(DocumentsOnlyInput.of(inferenceInputs), model); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java index 803bae40b33ed..6e006fe255956 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java @@ -110,8 +110,6 @@ public class OpenAiStreamingProcessor extends DelegatingProcessor parse(XContentParserConf ensureExpectedToken(XContentParser.Token.START_OBJECT, currentToken, parser); currentToken = parser.nextToken(); - if (currentToken == XContentParser.Token.END_OBJECT) { - consumeUntilObjectEnd(parser); // end choices - return ""; // stopped - } - if (currentToken == XContentParser.Token.FIELD_NAME && parser.currentName().equals(CONTENT_FIELD)) { - parser.nextToken(); - } else { - positionParserAtTokenAfterField(parser, CONTENT_FIELD, FAILED_TO_FIND_FIELD_TEMPLATE); + // continue until the end of delta + while (currentToken != null && currentToken != XContentParser.Token.END_OBJECT) { + if (currentToken == XContentParser.Token.START_OBJECT || currentToken == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } + + if (currentToken == XContentParser.Token.FIELD_NAME && parser.currentName().equals(CONTENT_FIELD)) { + parser.nextToken(); + ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.currentToken(), parser); + var content = parser.text(); + consumeUntilObjectEnd(parser); // end delta + consumeUntilObjectEnd(parser); // end choices + return content; + } + + currentToken = parser.nextToken(); } - ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.currentToken(), parser); - var content = parser.text(); - consumeUntilObjectEnd(parser); // end delta + consumeUntilObjectEnd(parser); // end choices - return content; + return ""; // stopped }).stream() .filter(Objects::nonNull) .filter(Predicate.not(String::isEmpty)) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntity.java deleted file mode 100644 index aff01316838f8..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntity.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; - -import org.elasticsearch.core.Nullable; - -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; - -public record AmazonBedrockAI21LabsCompletionRequestEntity( - List messages, - @Nullable Double temperature, - @Nullable Double topP, - @Nullable Integer maxTokenCount -) implements AmazonBedrockConverseRequestEntity { - - public AmazonBedrockAI21LabsCompletionRequestEntity { - Objects.requireNonNull(messages); - } - - @Override - public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) { - return request.messages(getConverseMessageList(messages)); - } - - @Override - public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) { - if (temperature == null && topP == null && maxTokenCount == null) { - return request; - } - - return request.inferenceConfig(config -> { - if (temperature != null) { - config.temperature(temperature.floatValue()); - } - - if (topP != null) { - config.topP(topP.floatValue()); - } - - if (maxTokenCount != null) { - config.maxTokens(maxTokenCount); - } - }); - } - - @Override - public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) { - return request; - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntity.java deleted file mode 100644 index 540012c221192..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntity.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; - -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Strings; - -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; - -public record AmazonBedrockAnthropicCompletionRequestEntity( - List messages, - @Nullable Double temperature, - @Nullable Double topP, - @Nullable Double topK, - @Nullable Integer maxTokenCount -) implements AmazonBedrockConverseRequestEntity { - - public AmazonBedrockAnthropicCompletionRequestEntity { - Objects.requireNonNull(messages); - } - - @Override - public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) { - return request.messages(getConverseMessageList(messages)); - } - - @Override - public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) { - if (temperature == null && topP == null && maxTokenCount == null) { - return request; - } - - return request.inferenceConfig(config -> { - if (temperature != null) { - config.temperature(temperature.floatValue()); - } - - if (topP != null) { - config.topP(topP.floatValue()); - } - - if (maxTokenCount != null) { - config.maxTokens(maxTokenCount); - } - }); - } - - @Override - public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) { - if (topK == null) { - return request; - } - - String topKField = Strings.format("{\"top_k\":%f}", topK.floatValue()); - return request.additionalModelResponseFieldPaths(topKField); - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java index f86d2229d42ad..db902290ba0be 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactory.java @@ -12,6 +12,8 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.additionalTopK; + public final class AmazonBedrockChatCompletionEntityFactory { public static AmazonBedrockConverseRequestEntity createEntity(AmazonBedrockChatCompletionModel model, List messages) { Objects.requireNonNull(model); @@ -19,55 +21,21 @@ public static AmazonBedrockConverseRequestEntity createEntity(AmazonBedrockChatC var serviceSettings = model.getServiceSettings(); var taskSettings = model.getTaskSettings(); switch (serviceSettings.provider()) { - case AI21LABS -> { - return new AmazonBedrockAI21LabsCompletionRequestEntity( - messages, - taskSettings.temperature(), - taskSettings.topP(), - taskSettings.maxNewTokens() - ); - } - case AMAZONTITAN -> { - return new AmazonBedrockTitanCompletionRequestEntity( - messages, - taskSettings.temperature(), - taskSettings.topP(), - taskSettings.maxNewTokens() - ); - } - case ANTHROPIC -> { - return new AmazonBedrockAnthropicCompletionRequestEntity( + case AI21LABS, AMAZONTITAN, META -> { + return new AmazonBedrockConverseRequestEntity( messages, taskSettings.temperature(), taskSettings.topP(), - taskSettings.topK(), taskSettings.maxNewTokens() ); } - case COHERE -> { - return new AmazonBedrockCohereCompletionRequestEntity( + case ANTHROPIC, COHERE, MISTRAL -> { + return new AmazonBedrockConverseRequestEntity( messages, taskSettings.temperature(), taskSettings.topP(), - taskSettings.topK(), - taskSettings.maxNewTokens() - ); - } - case META -> { - return new AmazonBedrockMetaCompletionRequestEntity( - messages, - taskSettings.temperature(), - taskSettings.topP(), - taskSettings.maxNewTokens() - ); - } - case MISTRAL -> { - return new AmazonBedrockMistralCompletionRequestEntity( - messages, - taskSettings.temperature(), - taskSettings.topP(), - taskSettings.topK(), - taskSettings.maxNewTokens() + taskSettings.maxNewTokens(), + additionalTopK(taskSettings.topK()) ); } default -> { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java index 61e0504732462..05d7d90873a71 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionRequest.java @@ -8,7 +8,9 @@ package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamRequest; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.TaskType; @@ -20,19 +22,26 @@ import java.io.IOException; import java.util.Objects; +import java.util.concurrent.Flow; + +import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; +import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.inferenceConfig; public class AmazonBedrockChatCompletionRequest extends AmazonBedrockRequest { public static final String USER_ROLE = "user"; private final AmazonBedrockConverseRequestEntity requestEntity; private AmazonBedrockChatCompletionResponseListener listener; + private final boolean stream; public AmazonBedrockChatCompletionRequest( AmazonBedrockChatCompletionModel model, AmazonBedrockConverseRequestEntity requestEntity, - @Nullable TimeValue timeout + @Nullable TimeValue timeout, + boolean stream ) { super(model, timeout); this.requestEntity = Objects.requireNonNull(requestEntity); + this.stream = stream; } @Override @@ -52,10 +61,16 @@ public TaskType taskType() { } private ConverseRequest getConverseRequest() { - var converseRequest = ConverseRequest.builder().modelId(amazonBedrockModel.model()); - converseRequest = requestEntity.addMessages(converseRequest); - converseRequest = requestEntity.addInferenceConfig(converseRequest); - converseRequest = requestEntity.addAdditionalModelFields(converseRequest); + var converseRequest = ConverseRequest.builder() + .modelId(amazonBedrockModel.model()) + .messages(getConverseMessageList(requestEntity.messages())) + .additionalModelResponseFieldPaths(requestEntity.additionalModelFields()); + + inferenceConfig(requestEntity).ifPresent(converseRequest::inferenceConfig); + + if (requestEntity.additionalModelFields() != null) { + converseRequest.additionalModelResponseFieldPaths(requestEntity.additionalModelFields()); + } return converseRequest.build(); } @@ -66,4 +81,22 @@ public void executeChatCompletionRequest( this.listener = chatCompletionResponseListener; this.executeRequest(awsBedrockClient); } + + public Flow.Publisher executeStreamChatCompletionRequest(AmazonBedrockBaseClient awsBedrockClient) { + var converseStreamRequest = ConverseStreamRequest.builder() + .modelId(amazonBedrockModel.model()) + .messages(getConverseMessageList(requestEntity.messages())); + + inferenceConfig(requestEntity).ifPresent(converseStreamRequest::inferenceConfig); + + if (requestEntity.additionalModelFields() != null) { + converseStreamRequest.additionalModelResponseFieldPaths(requestEntity.additionalModelFields()); + } + return awsBedrockClient.converseStream(converseStreamRequest.build()); + } + + @Override + public boolean isStreaming() { + return stream; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntity.java deleted file mode 100644 index f1ae04ad39516..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntity.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; - -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Strings; - -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; - -public record AmazonBedrockCohereCompletionRequestEntity( - List messages, - @Nullable Double temperature, - @Nullable Double topP, - @Nullable Double topK, - @Nullable Integer maxTokenCount -) implements AmazonBedrockConverseRequestEntity { - - public AmazonBedrockCohereCompletionRequestEntity { - Objects.requireNonNull(messages); - } - - @Override - public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) { - return request.messages(getConverseMessageList(messages)); - } - - @Override - public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) { - if (temperature == null && topP == null && maxTokenCount == null) { - return request; - } - - return request.inferenceConfig(config -> { - if (temperature != null) { - config.temperature(temperature.floatValue()); - } - - if (topP != null) { - config.topP(topP.floatValue()); - } - - if (maxTokenCount != null) { - config.maxTokens(maxTokenCount); - } - }); - } - - @Override - public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) { - if (topK == null) { - return request; - } - - String topKField = Strings.format("{\"top_k\":%f}", topK.floatValue()); - return request.additionalModelResponseFieldPaths(topKField); - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java index d8e9fa43797cd..203b2820ab16f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestEntity.java @@ -7,12 +7,23 @@ package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; -import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; +import org.elasticsearch.core.Nullable; -public interface AmazonBedrockConverseRequestEntity { - ConverseRequest.Builder addMessages(ConverseRequest.Builder request); +import java.util.List; - ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request); - - ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request); +public record AmazonBedrockConverseRequestEntity( + List messages, + @Nullable Double temperature, + @Nullable Double topP, + @Nullable Integer maxTokenCount, + @Nullable List additionalModelFields +) { + public AmazonBedrockConverseRequestEntity( + List messages, + @Nullable Double temperature, + @Nullable Double topP, + @Nullable Integer maxTokenCount + ) { + this(messages, temperature, topP, maxTokenCount, null); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java index 22e0d26a315a7..eb1652ff7ff6d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseUtils.java @@ -8,9 +8,14 @@ package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; import software.amazon.awssdk.services.bedrockruntime.model.ContentBlock; +import software.amazon.awssdk.services.bedrockruntime.model.InferenceConfiguration; import software.amazon.awssdk.services.bedrockruntime.model.Message; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; + import java.util.List; +import java.util.Optional; import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockChatCompletionRequest.USER_ROLE; @@ -22,4 +27,32 @@ public static List getConverseMessageList(List texts) { .map(content -> Message.builder().role(USER_ROLE).content(content).build()) .toList(); } + + public static Optional inferenceConfig(AmazonBedrockConverseRequestEntity request) { + if (request.temperature() != null || request.topP() != null || request.maxTokenCount() != null) { + var builder = InferenceConfiguration.builder(); + if (request.temperature() != null) { + builder.temperature(request.temperature().floatValue()); + } + + if (request.topP() != null) { + builder.topP(request.topP().floatValue()); + } + + if (request.maxTokenCount() != null) { + builder.maxTokens(request.maxTokenCount()); + } + return Optional.of(builder.build()); + } + return Optional.empty(); + } + + @Nullable + public static List additionalTopK(@Nullable Double topK) { + if (topK == null) { + return null; + } + + return List.of(Strings.format("{\"top_k\":%f}", topK.floatValue())); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntity.java deleted file mode 100644 index c21791ced02cb..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntity.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; - -import org.elasticsearch.core.Nullable; - -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; - -public record AmazonBedrockMetaCompletionRequestEntity( - List messages, - @Nullable Double temperature, - @Nullable Double topP, - @Nullable Integer maxTokenCount -) implements AmazonBedrockConverseRequestEntity { - - public AmazonBedrockMetaCompletionRequestEntity { - Objects.requireNonNull(messages); - } - - @Override - public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) { - return request.messages(getConverseMessageList(messages)); - } - - @Override - public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) { - if (temperature == null && topP == null && maxTokenCount == null) { - return request; - } - - return request.inferenceConfig(config -> { - if (temperature != null) { - config.temperature(temperature.floatValue()); - } - - if (topP != null) { - config.topP(topP.floatValue()); - } - - if (maxTokenCount != null) { - config.maxTokens(maxTokenCount); - } - }); - } - - @Override - public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) { - return request; - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntity.java deleted file mode 100644 index 15931674cbabb..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntity.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; - -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Strings; - -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; - -public record AmazonBedrockMistralCompletionRequestEntity( - List messages, - @Nullable Double temperature, - @Nullable Double topP, - @Nullable Double topK, - @Nullable Integer maxTokenCount -) implements AmazonBedrockConverseRequestEntity { - - public AmazonBedrockMistralCompletionRequestEntity { - Objects.requireNonNull(messages); - } - - @Override - public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) { - return request.messages(getConverseMessageList(messages)); - } - - @Override - public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) { - if (temperature == null && topP == null && maxTokenCount == null) { - return request; - } - - return request.inferenceConfig(config -> { - if (temperature != null) { - config.temperature(temperature.floatValue()); - } - - if (topP != null) { - config.topP(topP.floatValue()); - } - - if (maxTokenCount != null) { - config.maxTokens(maxTokenCount); - } - }); - } - - @Override - public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) { - if (topK == null) { - return request; - } - - String topKField = Strings.format("{\"top_k\":%f}", topK.floatValue()); - return request.additionalModelResponseFieldPaths(topKField); - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntity.java deleted file mode 100644 index e267592dfd0ba..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntity.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest; - -import org.elasticsearch.core.Nullable; - -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; - -public record AmazonBedrockTitanCompletionRequestEntity( - List messages, - @Nullable Double temperature, - @Nullable Double topP, - @Nullable Integer maxTokenCount -) implements AmazonBedrockConverseRequestEntity { - - public AmazonBedrockTitanCompletionRequestEntity { - Objects.requireNonNull(messages); - } - - @Override - public ConverseRequest.Builder addMessages(ConverseRequest.Builder request) { - return request.messages(getConverseMessageList(messages)); - } - - @Override - public ConverseRequest.Builder addInferenceConfig(ConverseRequest.Builder request) { - if (temperature == null && topP == null && maxTokenCount == null) { - return request; - } - - return request.inferenceConfig(config -> { - if (temperature != null) { - config.temperature(temperature.floatValue()); - } - - if (topP != null) { - config.topP(topP.floatValue()); - } - - if (maxTokenCount != null) { - config.maxTokens(maxTokenCount); - } - }); - } - - @Override - public ConverseRequest.Builder addAdditionalModelFields(ConverseRequest.Builder request) { - return request; - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java index f52fe623e7918..80770d63ef139 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java @@ -11,46 +11,63 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.ByteArrayEntity; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.request.HttpRequest; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.List; import java.util.Objects; public class GoogleAiStudioCompletionRequest implements GoogleAiStudioRequest { + private static final String ALT_PARAM = "alt"; + private static final String SSE_VALUE = "sse"; - private final List input; + private final DocumentsOnlyInput input; - private final URI uri; + private final LazyInitializable uri; private final GoogleAiStudioCompletionModel model; - public GoogleAiStudioCompletionRequest(List input, GoogleAiStudioCompletionModel model) { - this.input = input; + public GoogleAiStudioCompletionRequest(DocumentsOnlyInput input, GoogleAiStudioCompletionModel model) { + this.input = Objects.requireNonNull(input); this.model = Objects.requireNonNull(model); - this.uri = model.uri(); + this.uri = new LazyInitializable<>(() -> model.uri(input.stream())); } @Override public HttpRequest createHttpRequest() { - var httpPost = new HttpPost(uri); - var requestEntity = Strings.toString(new GoogleAiStudioCompletionRequestEntity(input)); + var httpPost = createHttpPost(); + var requestEntity = Strings.toString(new GoogleAiStudioCompletionRequestEntity(input.getInputs())); ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8)); httpPost.setEntity(byteEntity); httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()); - GoogleAiStudioRequest.decorateWithApiKeyParameter(httpPost, model.getSecretSettings()); return new HttpRequest(httpPost, getInferenceEntityId()); } + private HttpPost createHttpPost() { + try { + var uriBuilder = GoogleAiStudioRequest.builderWithApiKeyParameter(uri.getOrCompute(), model.getSecretSettings()); + if (isStreaming()) { + uriBuilder.addParameter(ALT_PARAM, SSE_VALUE); + } + return new HttpPost(uriBuilder.build()); + } catch (Exception e) { + ValidationException validationException = new ValidationException(e); + validationException.addValidationError(e.getMessage()); + throw validationException; + } + } + @Override public URI getURI() { - return this.uri; + return uri.getOrCompute(); } @Override @@ -69,4 +86,9 @@ public boolean[] getTruncationInfo() { public String getInferenceEntityId() { return model.getInferenceEntityId(); } + + @Override + public boolean isStreaming() { + return input.stream(); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java index fb99deabc9c5e..45403fb8e507d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java @@ -13,20 +13,15 @@ import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import java.net.URI; + public interface GoogleAiStudioRequest extends Request { String API_KEY_PARAMETER = "key"; static void decorateWithApiKeyParameter(HttpPost httpPost, DefaultSecretSettings secretSettings) { try { - var uri = httpPost.getURI(); - var uriWithApiKey = new URIBuilder().setScheme(uri.getScheme()) - .setHost(uri.getHost()) - .setPort(uri.getPort()) - .setPath(uri.getPath()) - .addParameter(API_KEY_PARAMETER, secretSettings.apiKey().toString()) - .build(); - + var uriWithApiKey = builderWithApiKeyParameter(httpPost.getURI(), secretSettings).build(); httpPost.setURI(uriWithApiKey); } catch (Exception e) { ValidationException validationException = new ValidationException(e); @@ -35,4 +30,12 @@ static void decorateWithApiKeyParameter(HttpPost httpPost, DefaultSecretSettings } } + static URIBuilder builderWithApiKeyParameter(URI uri, DefaultSecretSettings secretSettings) { + return new URIBuilder().setScheme(uri.getScheme()) + .setHost(uri.getHost()) + .setPort(uri.getPort()) + .setPath(uri.getPath()) + .addParameter(API_KEY_PARAMETER, secretSettings.apiKey().toString()); + } + } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java index 81ad5b6203682..16c9e7254e108 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java @@ -17,6 +17,8 @@ public class GoogleAiStudioUtils { public static final String GENERATE_CONTENT_ACTION = "generateContent"; + public static final String STREAM_GENERATE_CONTENT_ACTION = "streamGenerateContent"; + public static final String BATCH_EMBED_CONTENTS_ACTION = "batchEmbedContents"; private GoogleAiStudioUtils() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java index 852f25705d6ff..11dddc78bc469 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioCompletionResponseEntity.java @@ -77,33 +77,36 @@ public class GoogleAiStudioCompletionResponseEntity { public static ChatCompletionResults fromResponse(Request request, HttpResult response) throws IOException { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { - moveToFirstToken(jsonParser); + return new ChatCompletionResults(List.of(new ChatCompletionResults.Result(content(jsonParser)))); + } + } - XContentParser.Token token = jsonParser.currentToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + public static String content(XContentParser jsonParser) throws IOException { + moveToFirstToken(jsonParser); - positionParserAtTokenAfterField(jsonParser, "candidates", FAILED_TO_FIND_FIELD_TEMPLATE); + XContentParser.Token token = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); - jsonParser.nextToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, jsonParser.currentToken(), jsonParser); + positionParserAtTokenAfterField(jsonParser, "candidates", FAILED_TO_FIND_FIELD_TEMPLATE); - positionParserAtTokenAfterField(jsonParser, "content", FAILED_TO_FIND_FIELD_TEMPLATE); + jsonParser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, jsonParser.currentToken(), jsonParser); - token = jsonParser.currentToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + positionParserAtTokenAfterField(jsonParser, "content", FAILED_TO_FIND_FIELD_TEMPLATE); - positionParserAtTokenAfterField(jsonParser, "parts", FAILED_TO_FIND_FIELD_TEMPLATE); + token = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); - jsonParser.nextToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + positionParserAtTokenAfterField(jsonParser, "parts", FAILED_TO_FIND_FIELD_TEMPLATE); - positionParserAtTokenAfterField(jsonParser, "text", FAILED_TO_FIND_FIELD_TEMPLATE); + jsonParser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); - XContentParser.Token contentToken = jsonParser.currentToken(); - ensureExpectedToken(XContentParser.Token.VALUE_STRING, contentToken, jsonParser); - String content = jsonParser.text(); + positionParserAtTokenAfterField(jsonParser, "text", FAILED_TO_FIND_FIELD_TEMPLATE); + + XContentParser.Token contentToken = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.VALUE_STRING, contentToken, jsonParser); + return jsonParser.text(); - return new ChatCompletionResults(List.of(new ChatCompletionResults.Result(content))); - } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java index 62571c13aebf4..33a97f1e91621 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java @@ -23,15 +23,19 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; @@ -57,6 +61,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -87,29 +92,33 @@ public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap) private static final Logger logger = LogManager.getLogger(ModelRegistry.class); private final OriginSettingClient client; - private Map defaultConfigs; + private final List defaultConfigIds; private final Set preventDeletionLock = Collections.newSetFromMap(new ConcurrentHashMap<>()); public ModelRegistry(Client client) { this.client = new OriginSettingClient(client, ClientHelper.INFERENCE_ORIGIN); - this.defaultConfigs = new HashMap<>(); + defaultConfigIds = new ArrayList<>(); } - public void addDefaultConfiguration(UnparsedModel serviceDefaultConfig) { - if (defaultConfigs.containsKey(serviceDefaultConfig.inferenceEntityId())) { + /** + * Set the default inference ids provided by the services + * @param defaultConfigIds The defaults + */ + public void addDefaultIds(InferenceService.DefaultConfigId defaultConfigIds) { + var matched = idMatchedDefault(defaultConfigIds.inferenceId(), this.defaultConfigIds); + if (matched.isPresent()) { throw new IllegalStateException( "Cannot add default endpoint to the inference endpoint registry with duplicate inference id [" - + serviceDefaultConfig.inferenceEntityId() + + defaultConfigIds.inferenceId() + "] declared by service [" - + serviceDefaultConfig.service() + + defaultConfigIds.service().name() + "]. The inference Id is already use by [" - + defaultConfigs.get(serviceDefaultConfig.inferenceEntityId()).service() + + matched.get().service().name() + "] service." ); } - - defaultConfigs.put(serviceDefaultConfig.inferenceEntityId(), serviceDefaultConfig); + this.defaultConfigIds.add(defaultConfigIds); } /** @@ -118,15 +127,15 @@ public void addDefaultConfiguration(UnparsedModel serviceDefaultConfig) { * @param listener Model listener */ public void getModelWithSecrets(String inferenceEntityId, ActionListener listener) { - if (defaultConfigs.containsKey(inferenceEntityId)) { - listener.onResponse(deepCopyDefaultConfig(defaultConfigs.get(inferenceEntityId))); - return; - } - ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { - // There should be a hit for the configurations and secrets + // There should be a hit for the configurations if (searchResponse.getHits().getHits().length == 0) { - delegate.onFailure(inferenceNotFoundException(inferenceEntityId)); + var maybeDefault = idMatchedDefault(inferenceEntityId, defaultConfigIds); + if (maybeDefault.isPresent()) { + getDefaultConfig(maybeDefault.get(), listener); + } else { + delegate.onFailure(inferenceNotFoundException(inferenceEntityId)); + } return; } @@ -149,15 +158,15 @@ public void getModelWithSecrets(String inferenceEntityId, ActionListener listener) { - if (defaultConfigs.containsKey(inferenceEntityId)) { - listener.onResponse(deepCopyDefaultConfig(defaultConfigs.get(inferenceEntityId))); - return; - } - ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { - // There should be a hit for the configurations and secrets + // There should be a hit for the configurations if (searchResponse.getHits().getHits().length == 0) { - delegate.onFailure(inferenceNotFoundException(inferenceEntityId)); + var maybeDefault = idMatchedDefault(inferenceEntityId, defaultConfigIds); + if (maybeDefault.isPresent()) { + getDefaultConfig(maybeDefault.get(), listener); + } else { + delegate.onFailure(inferenceNotFoundException(inferenceEntityId)); + } return; } @@ -188,29 +197,9 @@ private ResourceNotFoundException inferenceNotFoundException(String inferenceEnt */ public void getModelsByTaskType(TaskType taskType, ActionListener> listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { - var defaultConfigsForTaskType = defaultConfigs.values() - .stream() - .filter(m -> m.taskType() == taskType) - .map(ModelRegistry::deepCopyDefaultConfig) - .toList(); - - // Not an error if no models of this task_type - if (searchResponse.getHits().getHits().length == 0 && defaultConfigsForTaskType.isEmpty()) { - delegate.onResponse(List.of()); - return; - } - var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistry::unparsedModelFromMap).toList(); - - if (defaultConfigsForTaskType.isEmpty() == false) { - var allConfigs = new ArrayList(); - allConfigs.addAll(modelConfigs); - allConfigs.addAll(defaultConfigsForTaskType); - allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId)); - delegate.onResponse(allConfigs); - } else { - delegate.onResponse(modelConfigs); - } + var defaultConfigsForTaskType = taskTypeMatchedDefaults(taskType, defaultConfigIds); + addAllDefaultConfigsIfMissing(modelConfigs, defaultConfigsForTaskType, delegate); }); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(TASK_TYPE_FIELD, taskType.toString())); @@ -232,19 +221,8 @@ public void getModelsByTaskType(TaskType taskType, ActionListener> listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { - var defaults = defaultConfigs.values().stream().map(ModelRegistry::deepCopyDefaultConfig).toList(); - - if (searchResponse.getHits().getHits().length == 0 && defaults.isEmpty()) { - delegate.onResponse(List.of()); - return; - } - var foundConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistry::unparsedModelFromMap).toList(); - var allConfigs = new ArrayList(); - allConfigs.addAll(foundConfigs); - allConfigs.addAll(defaults); - allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId)); - delegate.onResponse(allConfigs); + addAllDefaultConfigsIfMissing(foundConfigs, defaultConfigIds, delegate); }); // In theory the index should only contain model config documents @@ -262,6 +240,67 @@ public void getAllModels(ActionListener> listener) { client.search(modelSearch, searchListener); } + private void addAllDefaultConfigsIfMissing( + List foundConfigs, + List matchedDefaults, + ActionListener> listener + ) { + var foundIds = foundConfigs.stream().map(UnparsedModel::inferenceEntityId).collect(Collectors.toSet()); + var missing = matchedDefaults.stream().filter(d -> foundIds.contains(d.inferenceId()) == false).toList(); + + if (missing.isEmpty()) { + listener.onResponse(foundConfigs); + } else { + var groupedListener = new GroupedActionListener( + missing.size(), + listener.delegateFailure((delegate, listOfModels) -> { + var allConfigs = new ArrayList(); + allConfigs.addAll(foundConfigs); + allConfigs.addAll(listOfModels); + allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId)); + delegate.onResponse(allConfigs); + }) + ); + + for (var required : missing) { + getDefaultConfig(required, groupedListener); + } + } + } + + private void getDefaultConfig(InferenceService.DefaultConfigId defaultConfig, ActionListener listener) { + defaultConfig.service().defaultConfigs(listener.delegateFailureAndWrap((delegate, models) -> { + boolean foundModel = false; + for (var m : models) { + if (m.getInferenceEntityId().equals(defaultConfig.inferenceId())) { + foundModel = true; + storeDefaultEndpoint(m, () -> listener.onResponse(modelToUnparsedModel(m))); + break; + } + } + + if (foundModel == false) { + listener.onFailure( + new IllegalStateException("Configuration not found for default inference id [" + defaultConfig.inferenceId() + "]") + ); + } + })); + } + + public void storeDefaultEndpoint(Model preconfigured, Runnable runAfter) { + var responseListener = ActionListener.wrap(success -> { + logger.debug("Added default inference endpoint [{}]", preconfigured.getInferenceEntityId()); + }, exception -> { + if (exception instanceof ResourceAlreadyExistsException) { + logger.debug("Default inference id [{}] already exists", preconfigured.getInferenceEntityId()); + } else { + logger.error("Failed to store default inference id [" + preconfigured.getInferenceEntityId() + "]", exception); + } + }); + + storeModel(preconfigured, ActionListener.runAfter(responseListener, runAfter)); + } + private ArrayList parseHitsAsModels(SearchHits hits) { var modelConfigs = new ArrayList(); for (var hit : hits) { @@ -578,60 +617,36 @@ private static IndexRequest createIndexRequest(String docId, String indexName, T } } - private QueryBuilder documentIdQuery(String inferenceEntityId) { - return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(Model.documentId(inferenceEntityId))); - } - - static UnparsedModel deepCopyDefaultConfig(UnparsedModel other) { - // Because the default config uses immutable maps - return new UnparsedModel( - other.inferenceEntityId(), - other.taskType(), - other.service(), - copySettingsMap(other.settings()), - copySecretsMap(other.secrets()) - ); - } - - @SuppressWarnings("unchecked") - static Map copySettingsMap(Map other) { - var result = new HashMap(); - - var serviceSettings = (Map) other.get(ModelConfigurations.SERVICE_SETTINGS); - if (serviceSettings != null) { - var copiedServiceSettings = copyMap1LevelDeep(serviceSettings); - result.put(ModelConfigurations.SERVICE_SETTINGS, copiedServiceSettings); - } + private static UnparsedModel modelToUnparsedModel(Model model) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + model.getConfigurations() + .toXContent(builder, new ToXContent.MapParams(Map.of(ModelConfigurations.USE_ID_FOR_INDEX, Boolean.TRUE.toString()))); - var taskSettings = (Map) other.get(ModelConfigurations.TASK_SETTINGS); - if (taskSettings != null) { - var copiedTaskSettings = copyMap1LevelDeep(taskSettings); - result.put(ModelConfigurations.TASK_SETTINGS, copiedTaskSettings); - } + var modelConfigMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + return unparsedModelFromMap(new ModelConfigMap(modelConfigMap, new HashMap<>())); - var chunkSettings = (Map) other.get(ModelConfigurations.CHUNKING_SETTINGS); - if (chunkSettings != null) { - var copiedChunkSettings = copyMap1LevelDeep(chunkSettings); - result.put(ModelConfigurations.CHUNKING_SETTINGS, copiedChunkSettings); + } catch (IOException ex) { + throw new ElasticsearchException("[{}] Error serializing inference endpoint configuration", model.getInferenceEntityId(), ex); } + } - return result; + private QueryBuilder documentIdQuery(String inferenceEntityId) { + return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(Model.documentId(inferenceEntityId))); } - static Map copySecretsMap(Map other) { - return copyMap1LevelDeep(other); + static Optional idMatchedDefault( + String inferenceId, + List defaultConfigIds + ) { + return defaultConfigIds.stream().filter(defaultConfigId -> defaultConfigId.inferenceId().equals(inferenceId)).findFirst(); } - @SuppressWarnings("unchecked") - static Map copyMap1LevelDeep(Map other) { - var result = new HashMap(); - for (var entry : other.entrySet()) { - if (entry.getValue() instanceof Map) { - result.put(entry.getKey(), new HashMap<>((Map) entry.getValue())); - } else { - result.put(entry.getKey(), entry.getValue()); - } - } - return result; + static List taskTypeMatchedDefaults( + TaskType taskType, + List defaultConfigIds + ) { + return defaultConfigIds.stream() + .filter(defaultConfigId -> defaultConfigId.taskType().equals(taskType)) + .collect(Collectors.toList()); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java index 7ae7c5f4c3909..a397da05b1ce4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Releasables; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Streams; import org.elasticsearch.rest.ChunkedRestResponseBodyPart; import org.elasticsearch.rest.RestChannel; @@ -299,9 +298,7 @@ private ServerSentEventResponseBodyPart(ServerSentEvents event, ChunkedToXConten this.xContentBuilder = new LazyInitializable<>( () -> channel.newBuilder(channel.request().getXContentType(), null, true, Streams.noCloseStream(out)) ); - this.serialization = channel.request().getRestApiVersion() == RestApiVersion.V_7 - ? item.toXContentChunkedV7(params) - : item.toXContentChunked(params); + this.serialization = item.toXContentChunked(channel.request().getRestApiVersion(), params); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index 9b066f2a1679a..e1ed23a318e6c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.TransportVersions.ML_INFERENCE_AMAZON_BEDROCK_ADDED; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; @@ -267,6 +268,11 @@ public TransportVersion getMinimalSupportedVersion() { return ML_INFERENCE_AMAZON_BEDROCK_ADDED; } + @Override + public Set supportedStreamingTasks() { + return COMPLETION_ONLY; + } + /** * For text embedding models get the embedding size and * update the service settings. diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java index 881e2e82b766a..98777e9722242 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceExtension; @@ -22,6 +23,7 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; @@ -38,7 +40,6 @@ import java.io.IOException; import java.util.EnumSet; import java.util.List; -import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; @@ -49,14 +50,21 @@ public abstract class BaseElasticsearchInternalService implements InferenceServi protected final OriginSettingClient client; protected final ExecutorService inferenceExecutor; - protected final Consumer>> platformArch; + protected final Consumer> preferredModelVariantFn; + private final ClusterService clusterService; + + public enum PreferredModelVariant { + LINUX_X86_OPTIMIZED, + PLATFORM_AGNOSTIC + }; private static final Logger logger = LogManager.getLogger(BaseElasticsearchInternalService.class); public BaseElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) { this.client = new OriginSettingClient(context.client(), ClientHelper.INFERENCE_ORIGIN); this.inferenceExecutor = context.threadPool().executor(InferencePlugin.UTILITY_THREAD_POOL_NAME); - this.platformArch = this::platformArchitecture; + this.preferredModelVariantFn = this::preferredVariantFromPlatformArchitecture; + this.clusterService = context.clusterService(); } // For testing. @@ -66,11 +74,12 @@ public BaseElasticsearchInternalService(InferenceServiceExtension.InferenceServi // service package. public BaseElasticsearchInternalService( InferenceServiceExtension.InferenceServiceFactoryContext context, - Consumer>> platformArchFn + Consumer> preferredModelVariantFn ) { this.client = new OriginSettingClient(context.client(), ClientHelper.INFERENCE_ORIGIN); this.inferenceExecutor = context.threadPool().executor(InferencePlugin.UTILITY_THREAD_POOL_NAME); - this.platformArch = platformArchFn; + this.preferredModelVariantFn = preferredModelVariantFn; + this.clusterService = context.clusterService(); } /** @@ -206,31 +215,36 @@ protected void isBuiltinModelPut(Model model, ActionListener listener) public void close() throws IOException {} public static String selectDefaultModelVariantBasedOnClusterArchitecture( - Set modelArchitectures, - String linuxX86OptimisedModel, + PreferredModelVariant preferredModelVariant, + String linuxX86OptimizedModel, String platformAgnosticModel ) { // choose a default model version based on the cluster architecture - boolean homogenous = modelArchitectures.size() == 1; - if (homogenous && modelArchitectures.iterator().next().equals("linux-x86_64")) { + if (PreferredModelVariant.LINUX_X86_OPTIMIZED.equals(preferredModelVariant)) { // Use the hardware optimized model - return linuxX86OptimisedModel; + return linuxX86OptimizedModel; } else { // default to the platform-agnostic model return platformAgnosticModel; } } - private void platformArchitecture(ActionListener> platformArchitectureListener) { + private void preferredVariantFromPlatformArchitecture(ActionListener preferredVariantListener) { // Find the cluster platform as the service may need that // information when creating the model MlPlatformArchitecturesUtil.getMlNodesArchitecturesSet( - platformArchitectureListener.delegateFailureAndWrap((delegate, architectures) -> { - if (architectures.isEmpty() && clusterIsInElasticCloud()) { - // In Elastic cloud ml nodes run on Linux x86 - delegate.onResponse(Set.of("linux-x86_64")); + preferredVariantListener.delegateFailureAndWrap((delegate, architectures) -> { + if (architectures.isEmpty() && isClusterInElasticCloud()) { + // There are no ml nodes to check the current arch. + // However, in Elastic cloud ml nodes run on Linux x86 + delegate.onResponse(PreferredModelVariant.LINUX_X86_OPTIMIZED); } else { - delegate.onResponse(architectures); + boolean homogenous = architectures.size() == 1; + if (homogenous && architectures.iterator().next().equals("linux-x86_64")) { + delegate.onResponse(PreferredModelVariant.LINUX_X86_OPTIMIZED); + } else { + delegate.onResponse(PreferredModelVariant.PLATFORM_AGNOSTIC); + } } }), client, @@ -238,9 +252,11 @@ private void platformArchitecture(ActionListener> platformArchitectu ); } - static boolean clusterIsInElasticCloud() { - // use a heuristic to determine if in Elastic cloud. - return true; // TODO + boolean isClusterInElasticCloud() { + // Use the ml lazy node count as a heuristic to determine if in Elastic cloud. + // A value > 0 means scaling should be available for ml nodes + var maxMlLazyNodes = clusterService.getClusterSettings().get(MachineLearningField.MAX_LAZY_ML_NODES); + return maxMlLazyNodes > 0; } public static InferModelAction.Request buildInferenceRequest( @@ -259,7 +275,7 @@ public static InferModelAction.Request buildInferenceRequest( return request; } - protected abstract boolean isDefaultId(String inferenceId); + abstract boolean isDefaultId(String inferenceId); protected void maybeStartDeployment( ElasticsearchInternalModel model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java index 2aee37312a16e..f312790ded655 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java @@ -21,7 +21,7 @@ public abstract class ElasticsearchInternalModel extends Model { - protected final ElasticsearchInternalServiceSettings internalServiceSettings; + protected ElasticsearchInternalServiceSettings internalServiceSettings; public ElasticsearchInternalModel( String inferenceEntityId, @@ -87,6 +87,15 @@ public ElasticsearchInternalServiceSettings getServiceSettings() { return (ElasticsearchInternalServiceSettings) super.getServiceSettings(); } + public void updateNumAllocation(Integer numAllocations) { + this.internalServiceSettings = new ElasticsearchInternalServiceSettings( + numAllocations, + this.internalServiceSettings.getNumThreads(), + this.internalServiceSettings.modelId(), + this.internalServiceSettings.getAdaptiveAllocationsSettings() + ); + } + @Override public String toString() { return Strings.toString(this.getConfigurations()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 9a4201842873e..396c679bf322e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -27,14 +27,15 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; @@ -50,6 +51,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -62,7 +64,6 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL_LINUX_X86; -import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.EMBEDDING_MAX_BATCH_SIZE; public class ElasticsearchInternalService extends BaseElasticsearchInternalService { @@ -78,6 +79,7 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi public static final int EMBEDDING_MAX_BATCH_SIZE = 10; public static final String DEFAULT_ELSER_ID = ".elser-2"; + public static final String DEFAULT_E5_ID = ".multi-e5-small"; private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class); private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(ElasticsearchInternalService.class); @@ -89,7 +91,7 @@ public ElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFa // for testing ElasticsearchInternalService( InferenceServiceExtension.InferenceServiceFactoryContext context, - Consumer>> platformArch + Consumer> platformArch ) { super(context, platformArch); } @@ -145,14 +147,15 @@ public void parseRequestConfig( "Putting elasticsearch service inference endpoints (including elser service) without a model_id field is" + " deprecated and will be removed in a future release. Please specify a model_id field." ); - platformArch.accept( + preferredModelVariantFn.accept( modelListener.delegateFailureAndWrap( - (delegate, arch) -> elserCase( + (delegate, preferredModelVariant) -> elserCase( inferenceEntityId, taskType, config, - arch, + preferredModelVariant, serviceSettingsMap, + true, chunkingSettings, modelListener ) @@ -162,13 +165,13 @@ public void parseRequestConfig( throw new IllegalArgumentException("Error parsing service settings, model_id must be provided"); } } else if (MULTILINGUAL_E5_SMALL_VALID_IDS.contains(modelId)) { - platformArch.accept( + preferredModelVariantFn.accept( modelListener.delegateFailureAndWrap( - (delegate, arch) -> e5Case( + (delegate, preferredModelVariant) -> e5Case( inferenceEntityId, taskType, config, - arch, + preferredModelVariant, serviceSettingsMap, chunkingSettings, modelListener @@ -176,14 +179,15 @@ public void parseRequestConfig( ) ); } else if (ElserModels.isValidModel(modelId)) { - platformArch.accept( + preferredModelVariantFn.accept( modelListener.delegateFailureAndWrap( - (delegate, arch) -> elserCase( + (delegate, preferredModelVariant) -> elserCase( inferenceEntityId, taskType, config, - arch, + preferredModelVariant, serviceSettingsMap, + OLD_ELSER_SERVICE_NAME.equals(serviceName), chunkingSettings, modelListener ) @@ -286,7 +290,7 @@ private void e5Case( String inferenceEntityId, TaskType taskType, Map config, - Set platformArchitectures, + PreferredModelVariant preferredModelVariant, Map serviceSettingsMap, ChunkingSettings chunkingSettings, ActionListener modelListener @@ -296,12 +300,12 @@ private void e5Case( if (esServiceSettingsBuilder.getModelId() == null) { esServiceSettingsBuilder.setModelId( selectDefaultModelVariantBasedOnClusterArchitecture( - platformArchitectures, + preferredModelVariant, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86, MULTILINGUAL_E5_SMALL_MODEL_ID ) ); - } else if (modelVariantValidForArchitecture(platformArchitectures, esServiceSettingsBuilder.getModelId()) == false) { + } else if (modelVariantValidForArchitecture(preferredModelVariant, esServiceSettingsBuilder.getModelId()) == false) { throw new IllegalArgumentException( "Error parsing request config, model id does not match any models available on this platform. Was [" + esServiceSettingsBuilder.getModelId() @@ -323,14 +327,14 @@ private void e5Case( ); } - static boolean modelVariantValidForArchitecture(Set platformArchitectures, String modelId) { + static boolean modelVariantValidForArchitecture(PreferredModelVariant modelVariant, String modelId) { if (modelId.equals(MULTILINGUAL_E5_SMALL_MODEL_ID)) { // platform agnostic model is always compatible return true; } return modelId.equals( selectDefaultModelVariantBasedOnClusterArchitecture( - platformArchitectures, + modelVariant, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86, MULTILINGUAL_E5_SMALL_MODEL_ID ) @@ -341,14 +345,15 @@ private void elserCase( String inferenceEntityId, TaskType taskType, Map config, - Set platformArchitectures, + PreferredModelVariant preferredModelVariant, Map serviceSettingsMap, + boolean isElserService, ChunkingSettings chunkingSettings, ActionListener modelListener ) { var esServiceSettingsBuilder = ElasticsearchInternalServiceSettings.fromRequestMap(serviceSettingsMap); final String defaultModelId = selectDefaultModelVariantBasedOnClusterArchitecture( - platformArchitectures, + preferredModelVariant, ELSER_V2_MODEL_LINUX_X86, ELSER_V2_MODEL ); @@ -373,21 +378,15 @@ private void elserCase( } } - DEPRECATION_LOGGER.warn( - DeprecationCategory.API, - "inference_api_elser_service", - "The [{}] service is deprecated and will be removed in a future release. Use the [{}] service instead, with" - + " [model_id] set to [{}] in the [service_settings]", - OLD_ELSER_SERVICE_NAME, - ElasticsearchInternalService.NAME, - defaultModelId - ); - - if (modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic(platformArchitectures, esServiceSettingsBuilder.getModelId())) { - throw new IllegalArgumentException( - "Error parsing request config, model id does not match any models available on this platform. Was [" - + esServiceSettingsBuilder.getModelId() - + "]" + if (isElserService) { + DEPRECATION_LOGGER.warn( + DeprecationCategory.API, + "inference_api_elser_service", + "The [{}] service is deprecated and will be removed in a future release. Use the [{}] service instead, with" + + " [model_id] set to [{}] in the [service_settings]", + OLD_ELSER_SERVICE_NAME, + ElasticsearchInternalService.NAME, + defaultModelId ); } @@ -406,19 +405,6 @@ private void elserCase( ); } - private static boolean modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic( - Set platformArchitectures, - String modelId - ) { - return modelId.equals( - selectDefaultModelVariantBasedOnClusterArchitecture( - platformArchitectures, - MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86, - MULTILINGUAL_E5_SMALL_MODEL_ID - ) - ); - } - @Override public Model parsePersistedConfigWithSecrets( String inferenceEntityId, @@ -793,42 +779,104 @@ private RankedDocsResults textSimilarityResultsToRankedDocs( return new RankedDocsResults(rankings); } + public List defaultConfigIds() { + return List.of( + new DefaultConfigId(DEFAULT_ELSER_ID, TaskType.SPARSE_EMBEDDING, this), + new DefaultConfigId(DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, this) + ); + } + @Override - public List defaultConfigs() { - // TODO Chunking settings - Map elserSettings = Map.of( - ModelConfigurations.SERVICE_SETTINGS, - Map.of( - ElasticsearchInternalServiceSettings.MODEL_ID, - ElserModels.ELSER_V2_MODEL, // TODO pick model depending on platform - ElasticsearchInternalServiceSettings.NUM_THREADS, - 1, - ElasticsearchInternalServiceSettings.ADAPTIVE_ALLOCATIONS, - Map.of( - "enabled", - Boolean.TRUE, - "min_number_of_allocations", - 1, - "max_number_of_allocations", - 8 // no max? - ) - ) + public void updateModelsWithDynamicFields(List models, ActionListener> listener) { + var modelsByDeploymentIds = new HashMap(); + for (var model : models) { + if (model instanceof ElasticsearchInternalModel esModel) { + modelsByDeploymentIds.put(esModel.internalServiceSettings.deloymentId(), esModel); + } else { + listener.onFailure( + new ElasticsearchStatusException( + "Cannot update model [{}] as it is not an Elasticsearch service model", + RestStatus.INTERNAL_SERVER_ERROR, + model.getInferenceEntityId() + ) + ); + return; + } + } + + if (modelsByDeploymentIds.isEmpty()) { + listener.onResponse(models); + return; + } + + String deploymentIds = String.join(",", modelsByDeploymentIds.keySet()); + client.execute( + GetDeploymentStatsAction.INSTANCE, + new GetDeploymentStatsAction.Request(deploymentIds), + ActionListener.wrap(stats -> { + for (var deploymentStats : stats.getStats().results()) { + var model = modelsByDeploymentIds.get(deploymentStats.getDeploymentId()); + model.updateNumAllocation(deploymentStats.getNumberOfAllocations()); + } + listener.onResponse(new ArrayList<>(modelsByDeploymentIds.values())); + }, e -> { + logger.warn("Get deployment stats failed, cannot update the endpoint's number of allocations", e); + // continue with the original response + listener.onResponse(models); + }) ); + } - return List.of( - new UnparsedModel( - DEFAULT_ELSER_ID, - TaskType.SPARSE_EMBEDDING, - NAME, - elserSettings, - Map.of() // no secrets - ) + public void defaultConfigs(ActionListener> defaultsListener) { + preferredModelVariantFn.accept(defaultsListener.delegateFailureAndWrap((delegate, preferredModelVariant) -> { + if (PreferredModelVariant.LINUX_X86_OPTIMIZED.equals(preferredModelVariant)) { + defaultsListener.onResponse(defaultConfigsLinuxOptimized()); + } else { + defaultsListener.onResponse(defaultConfigsPlatfromAgnostic()); + } + })); + } + + private List defaultConfigsLinuxOptimized() { + return defaultConfigs(true); + } + + private List defaultConfigsPlatfromAgnostic() { + return defaultConfigs(false); + } + + private List defaultConfigs(boolean useLinuxOptimizedModel) { + var defaultElser = new ElserInternalModel( + DEFAULT_ELSER_ID, + TaskType.SPARSE_EMBEDDING, + NAME, + new ElserInternalServiceSettings( + null, + 1, + useLinuxOptimizedModel ? ELSER_V2_MODEL_LINUX_X86 : ELSER_V2_MODEL, + new AdaptiveAllocationsSettings(Boolean.TRUE, 1, 8) + ), + ElserMlNodeTaskSettings.DEFAULT, + null // default chunking settings + ); + var defaultE5 = new MultilingualE5SmallModel( + DEFAULT_E5_ID, + TaskType.TEXT_EMBEDDING, + NAME, + new MultilingualE5SmallInternalServiceSettings( + null, + 1, + useLinuxOptimizedModel ? MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 : MULTILINGUAL_E5_SMALL_MODEL_ID, + new AdaptiveAllocationsSettings(Boolean.TRUE, 1, 8) + ), + null // default chunking settings ); + return List.of(defaultElser, defaultE5); } @Override - protected boolean isDefaultId(String inferenceId) { - return DEFAULT_ELSER_ID.equals(inferenceId); + boolean isDefaultId(String inferenceId) { + return DEFAULT_ELSER_ID.equals(inferenceId) || DEFAULT_E5_ID.equals(inferenceId); } static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSettings( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index 37e0f28dfb3fe..68db964e86b10 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -166,6 +166,10 @@ public String modelId() { return modelId; } + public String deloymentId() { + return modelId; + } + public Integer getNumAllocations() { return numAllocations; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java index d817a3bbb73ef..d29095be808b9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java @@ -7,15 +7,11 @@ package org.elasticsearch.xpack.inference.services.googleaistudio; -import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.ServiceSettings; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor; -import java.util.Map; import java.util.Objects; public abstract class GoogleAiStudioModel extends Model { @@ -38,8 +34,6 @@ public GoogleAiStudioModel(GoogleAiStudioModel model, ServiceSettings serviceSet rateLimitServiceSettings = model.rateLimitServiceSettings(); } - public abstract ExecutableAction accept(GoogleAiStudioActionVisitor creator, Map taskSettings, InputType inputType); - public GoogleAiStudioRateLimitServiceSettings rateLimitServiceSettings() { return rateLimitServiceSettings; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java index 798ce4ea5800b..c685441271194 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java @@ -27,8 +27,11 @@ import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; -import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionCreator; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; @@ -42,7 +45,9 @@ import java.util.List; import java.util.Map; +import java.util.Set; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; @@ -211,6 +216,11 @@ public TransportVersion getMinimalSupportedVersion() { return TransportVersions.ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED; } + @Override + public Set supportedStreamingTasks() { + return COMPLETION_ONLY; + } + @Override public void checkModelConfig(Model model, ActionListener listener) { // TODO: Remove this function once all services have been updated to use the new model validators @@ -247,16 +257,32 @@ protected void doInfer( TimeValue timeout, ActionListener listener ) { - if (model instanceof GoogleAiStudioModel == false) { + if (model instanceof GoogleAiStudioCompletionModel completionModel) { + var requestManager = new GoogleAiStudioCompletionRequestManager(completionModel, getServiceComponents().threadPool()); + var docsOnly = DocumentsOnlyInput.of(inputs); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( + completionModel.uri(docsOnly.stream()), + "Google AI Studio completion" + ); + var action = new SingleInputSenderExecutableAction( + getSender(), + requestManager, + failedToSendRequestErrorMessage, + "Google AI Studio completion" + ); + action.execute(inputs, timeout, listener); + } else if (model instanceof GoogleAiStudioEmbeddingsModel embeddingsModel) { + var requestManager = new GoogleAiStudioEmbeddingsRequestManager( + embeddingsModel, + getServiceComponents().truncator(), + getServiceComponents().threadPool() + ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(embeddingsModel.uri(), "Google AI Studio embeddings"); + var action = new SenderExecutableAction(getSender(), requestManager, failedToSendRequestErrorMessage); + action.execute(inputs, timeout, listener); + } else { listener.onFailure(createInvalidModelException(model)); - return; } - - GoogleAiStudioModel googleAiStudioModel = (GoogleAiStudioModel) model; - var actionCreator = new GoogleAiStudioActionCreator(getSender(), getServiceComponents()); - - var action = googleAiStudioModel.accept(actionCreator, taskSettings, inputType); - action.execute(inputs, timeout, listener); } @Override @@ -270,7 +296,6 @@ protected void doChunkedInfer( ActionListener> listener ) { GoogleAiStudioModel googleAiStudioModel = (GoogleAiStudioModel) model; - var actionCreator = new GoogleAiStudioActionCreator(getSender(), getServiceComponents()); List batchedRequests; if (ChunkingSettingsFeatureFlag.isEnabled()) { @@ -287,10 +312,8 @@ protected void doChunkedInfer( EmbeddingRequestChunker.EmbeddingType.FLOAT ).batchRequestsWithListeners(listener); } - for (var request : batchedRequests) { - var action = googleAiStudioModel.accept(actionCreator, taskSettings, inputType); - action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); + doInfer(model, new DocumentsOnlyInput(request.batch().inputs()), taskSettings, inputType, timeout, request.listener()); } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java index 8fa2ac0148716..7b793ab37d342 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java @@ -10,13 +10,10 @@ import org.apache.http.client.utils.URIBuilder; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.EmptyTaskSettings; -import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor; import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel; @@ -30,8 +27,6 @@ public class GoogleAiStudioCompletionModel extends GoogleAiStudioModel { - private URI uri; - public GoogleAiStudioCompletionModel( String inferenceEntityId, TaskType taskType, @@ -65,39 +60,20 @@ public GoogleAiStudioCompletionModel( new ModelSecrets(secrets), serviceSettings ); - try { - this.uri = buildUri(serviceSettings.modelId()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } } - // Should only be used directly for testing - GoogleAiStudioCompletionModel( - String inferenceEntityId, - TaskType taskType, - String service, - String url, - GoogleAiStudioCompletionServiceSettings serviceSettings, - TaskSettings taskSettings, - @Nullable DefaultSecretSettings secrets - ) { - super( - new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), - new ModelSecrets(secrets), - serviceSettings - ); + public URI uri(boolean streaming) { try { - this.uri = new URI(url); + var api = streaming ? GoogleAiStudioUtils.STREAM_GENERATE_CONTENT_ACTION : GoogleAiStudioUtils.GENERATE_CONTENT_ACTION; + return new URIBuilder().setScheme("https") + .setHost(GoogleAiStudioUtils.HOST_SUFFIX) + .setPathSegments(GoogleAiStudioUtils.V1, GoogleAiStudioUtils.MODELS, format("%s:%s", getServiceSettings().modelId(), api)) + .build(); } catch (URISyntaxException e) { throw new RuntimeException(e); } } - public URI uri() { - return uri; - } - @Override public GoogleAiStudioCompletionServiceSettings getServiceSettings() { return (GoogleAiStudioCompletionServiceSettings) super.getServiceSettings(); @@ -108,7 +84,8 @@ public DefaultSecretSettings getSecretSettings() { return (DefaultSecretSettings) super.getSecretSettings(); } - public static URI buildUri(String model) throws URISyntaxException { + // visible for testing + static URI buildUri(String model) throws URISyntaxException { return new URIBuilder().setScheme("https") .setHost(GoogleAiStudioUtils.HOST_SUFFIX) .setPathSegments( @@ -118,9 +95,4 @@ public static URI buildUri(String model) throws URISyntaxException { ) .build(); } - - @Override - public ExecutableAction accept(GoogleAiStudioActionVisitor visitor, Map taskSettings, InputType inputType) { - return visitor.create(this, taskSettings); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java index 5d46a8e129dff..a9434fb473599 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java @@ -11,13 +11,10 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.EmptyTaskSettings; -import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor; import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel; @@ -139,11 +136,6 @@ public URI uri() { return uri; } - @Override - public ExecutableAction accept(GoogleAiStudioActionVisitor visitor, Map taskSettings, InputType inputType) { - return visitor.create(this, taskSettings); - } - public static URI buildUri(String model) throws URISyntaxException { return new URIBuilder().setScheme("https") .setHost(GoogleAiStudioUtils.HOST_SUFFIX) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java index 3c09984ac0162..5b9625073e6c6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java @@ -17,7 +17,7 @@ public class ChunkingSettingsBuilderTests extends ESTestCase { - public static final WordBoundaryChunkingSettings DEFAULT_SETTINGS = new WordBoundaryChunkingSettings(250, 100); + public static final SentenceBoundaryChunkingSettings DEFAULT_SETTINGS = new SentenceBoundaryChunkingSettings(250, 1); public void testEmptyChunkingSettingsMap() { ChunkingSettings chunkingSettings = ChunkingSettingsBuilder.fromMap(Collections.emptyMap()); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java index 6fcd386702497..72b5ffa45a0dd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java @@ -272,7 +272,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc private ExecutableAction createAction(String url, String apiKey, String modelName, Sender sender) { var model = GoogleAiStudioCompletionModelTests.createModel(modelName, url, apiKey); var requestManager = new GoogleAiStudioCompletionRequestManager(model, threadPool); - var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio completion"); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(false), "Google AI Studio completion"); return new SingleInputSenderExecutableAction( sender, requestManager, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java index 8f09c53c99366..6d601b4b08c53 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecutorTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockChatCompletionRequest; -import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockTitanCompletionRequestEntity; +import org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestEntity; import org.elasticsearch.xpack.inference.external.request.amazonbedrock.embeddings.AmazonBedrockEmbeddingsRequest; import org.elasticsearch.xpack.inference.external.request.amazonbedrock.embeddings.AmazonBedrockTitanEmbeddingsRequestEntity; import org.elasticsearch.xpack.inference.external.response.amazonbedrock.completion.AmazonBedrockChatCompletionResponseHandler; @@ -100,8 +100,8 @@ public void testExecute_ChatCompletionRequest() throws CharacterCodingException "secretkey" ); - var requestEntity = new AmazonBedrockTitanCompletionRequestEntity(List.of("abc"), null, null, 512); - var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null); + var requestEntity = new AmazonBedrockConverseRequestEntity(List.of("abc"), null, null, 512); + var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null, false); var responseHandler = new AmazonBedrockChatCompletionResponseHandler(); var clientCache = new AmazonBedrockMockClientCache(getTestConverseResult("converse result"), null, null); @@ -124,8 +124,8 @@ public void testExecute_FailsProperly_WithElasticsearchException() { "secretkey" ); - var requestEntity = new AmazonBedrockTitanCompletionRequestEntity(List.of("abc"), null, null, 512); - var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null); + var requestEntity = new AmazonBedrockConverseRequestEntity(List.of("abc"), null, null, 512); + var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, null, false); var responseHandler = new AmazonBedrockChatCompletionResponseHandler(); var clientCache = new AmazonBedrockMockClientCache(null, null, new ElasticsearchException("test exception")); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java index 873b2e22497c6..bb7c669cdf09b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockInferenceClientCacheTests.java @@ -25,7 +25,7 @@ public class AmazonBedrockInferenceClientCacheTests extends ESTestCase { public void testCache_ReturnsSameObject() throws IOException { AmazonBedrockInferenceClientCache cacheInstance; - try (var cache = new AmazonBedrockInferenceClientCache(AmazonBedrockMockInferenceClient::create, null)) { + try (var cache = new AmazonBedrockInferenceClientCache(AmazonBedrockMockInferenceClient::create, Clock.systemUTC())) { cacheInstance = cache; var model = AmazonBedrockEmbeddingsModelTests.createModel( "inferenceId", diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java index 5584e90b3264d..e6cd667b824b3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockInferenceClient.java @@ -14,15 +14,19 @@ import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockModel; import java.util.concurrent.CompletableFuture; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class AmazonBedrockMockInferenceClient extends AmazonBedrockInferenceClient { private CompletableFuture converseResponseFuture = CompletableFuture.completedFuture(null); @@ -33,7 +37,13 @@ public static AmazonBedrockMockInferenceClient create(AmazonBedrockModel model, } protected AmazonBedrockMockInferenceClient(AmazonBedrockModel model, @Nullable TimeValue timeout) { - super(model, timeout); + super(model, timeout, mockThreadPool()); + } + + private static ThreadPool mockThreadPool() { + ThreadPool threadPool = mock(); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + return threadPool; } public void setExceptionToThrow(ElasticsearchException exceptionToThrow) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessorTests.java new file mode 100644 index 0000000000000..ba87bdfe16cdd --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockStreamingChatProcessorTests.java @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.amazonbedrock; + +import software.amazon.awssdk.services.bedrockruntime.model.BedrockRuntimeException; +import software.amazon.awssdk.services.bedrockruntime.model.ContentBlockDelta; +import software.amazon.awssdk.services.bedrockruntime.model.ContentBlockDeltaEvent; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamOutput; +import software.amazon.awssdk.services.bedrockruntime.model.ConverseStreamResponseHandler; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.util.Arrays; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class AmazonBedrockStreamingChatProcessorTests extends ESTestCase { + private AmazonBedrockStreamingChatProcessor processor; + + @Before + public void setUp() throws Exception { + super.setUp(); + ThreadPool threadPool = mock(); + when(threadPool.executor(UTILITY_THREAD_POOL_NAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + processor = new AmazonBedrockStreamingChatProcessor(threadPool); + } + + /** + * We do not issue requests on subscribe because the downstream will control the pacing. + */ + public void testOnSubscribeBeforeDownstreamDoesNotRequest() { + var upstream = mock(Flow.Subscription.class); + processor.onSubscribe(upstream); + + verify(upstream, never()).request(anyLong()); + } + + /** + * If the downstream requests data before the upstream is set, when the upstream is set, we will forward the pending requests to it. + */ + public void testOnSubscribeAfterDownstreamRequests() { + var expectedRequestCount = randomLongBetween(1, 500); + Flow.Subscriber subscriber = mock(); + doAnswer(ans -> { + Flow.Subscription sub = ans.getArgument(0); + sub.request(expectedRequestCount); + return null; + }).when(subscriber).onSubscribe(any()); + processor.subscribe(subscriber); + + var upstream = mock(Flow.Subscription.class); + processor.onSubscribe(upstream); + + verify(upstream, times(1)).request(anyLong()); + } + + public void testCancelDuplicateSubscriptions() { + processor.onSubscribe(mock()); + + var upstream = mock(Flow.Subscription.class); + processor.onSubscribe(upstream); + + verify(upstream, times(1)).cancel(); + verifyNoMoreInteractions(upstream); + } + + public void testMultiplePublishesCallsOnError() { + processor.subscribe(mock()); + + Flow.Subscriber subscriber = mock(); + processor.subscribe(subscriber); + + verify(subscriber, times(1)).onError(assertArg(e -> { + assertThat(e, isA(IllegalStateException.class)); + assertThat(e.getMessage(), equalTo("Subscriber already set.")); + })); + } + + public void testNonDeltaBlocksAreSkipped() { + var upstream = mock(Flow.Subscription.class); + processor.onSubscribe(upstream); + var counter = new AtomicInteger(); + Arrays.stream(ConverseStreamOutput.EventType.values()) + .filter(type -> type != ConverseStreamOutput.EventType.CONTENT_BLOCK_DELTA) + .forEach(type -> { + ConverseStreamOutput output = mock(); + when(output.sdkEventType()).thenReturn(type); + processor.onNext(output); + verify(upstream, times(counter.incrementAndGet())).request(eq(1L)); + }); + } + + public void testDeltaBlockForwardsDownstream() { + var expectedText = "hello"; + + // mock executorservice so we can make sure we handle the response on another thread + ExecutorService executorService = mock(); + ThreadPool threadPool = mock(); + when(threadPool.executor(UTILITY_THREAD_POOL_NAME)).thenReturn(executorService); + processor = new AmazonBedrockStreamingChatProcessor(threadPool); + doAnswer(ans -> { + Runnable command = ans.getArgument(0); + command.run(); + return null; + }).when(executorService).execute(any()); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + ConverseStreamOutput output = output(expectedText); + + processor.onNext(output); + + verifyText(downstream, expectedText); + verify(executorService, times(1)).execute(any()); + verify(upstream, times(0)).request(anyLong()); + } + + private ConverseStreamOutput output(String text) { + ConverseStreamOutput output = mock(); + when(output.sdkEventType()).thenReturn(ConverseStreamOutput.EventType.CONTENT_BLOCK_DELTA); + doAnswer(ans -> { + ConverseStreamResponseHandler.Visitor visitor = ans.getArgument(0); + ContentBlockDelta delta = ContentBlockDelta.fromText(text); + ContentBlockDeltaEvent event = ContentBlockDeltaEvent.builder().delta(delta).build(); + visitor.visitContentBlockDelta(event); + return null; + }).when(output).accept(any()); + return output; + } + + private void verifyText(Flow.Subscriber downstream, String expectedText) { + verify(downstream, times(1)).onNext(assertArg(results -> { + assertThat(results, notNullValue()); + assertThat(results.results().size(), equalTo(1)); + assertThat(results.results().getFirst().delta(), equalTo(expectedText)); + })); + } + + public void verifyCompleteBeforeRequest() { + processor.onComplete(); + + Flow.Subscriber downstream = mock(); + var sub = ArgumentCaptor.forClass(Flow.Subscription.class); + processor.subscribe(downstream); + verify(downstream).onSubscribe(sub.capture()); + + sub.getValue().request(1); + verify(downstream, times(1)).onComplete(); + } + + public void verifyCompleteAfterRequest() { + + Flow.Subscriber downstream = mock(); + var sub = ArgumentCaptor.forClass(Flow.Subscription.class); + processor.subscribe(downstream); + verify(downstream).onSubscribe(sub.capture()); + + sub.getValue().request(1); + processor.onComplete(); + verify(downstream, times(1)).onComplete(); + } + + public void verifyOnErrorBeforeRequest() { + var expectedError = BedrockRuntimeException.builder().message("ahhhhhh").build(); + processor.onError(expectedError); + + Flow.Subscriber downstream = mock(); + var sub = ArgumentCaptor.forClass(Flow.Subscription.class); + processor.subscribe(downstream); + verify(downstream).onSubscribe(sub.capture()); + + sub.getValue().request(1); + verify(downstream, times(1)).onError(assertArg(e -> { + assertThat(e, isA(ElasticsearchException.class)); + assertThat(e.getCause(), is(expectedError)); + })); + } + + public void verifyOnErrorAfterRequest() { + var expectedError = BedrockRuntimeException.builder().message("ahhhhhh").build(); + + Flow.Subscriber downstream = mock(); + var sub = ArgumentCaptor.forClass(Flow.Subscription.class); + processor.subscribe(downstream); + verify(downstream).onSubscribe(sub.capture()); + + sub.getValue().request(1); + processor.onError(expectedError); + verify(downstream, times(1)).onError(assertArg(e -> { + assertThat(e, isA(ElasticsearchException.class)); + assertThat(e.getCause(), is(expectedError)); + })); + } + + public void verifyAsyncOnCompleteIsStillDeliveredSynchronously() { + mockUpstream(); + + Flow.Subscriber downstream = mock(); + var sub = ArgumentCaptor.forClass(Flow.Subscription.class); + processor.subscribe(downstream); + verify(downstream).onSubscribe(sub.capture()); + + sub.getValue().request(1); + verify(downstream, times(1)).onNext(any()); + processor.onComplete(); + verify(downstream, times(0)).onComplete(); + sub.getValue().request(1); + verify(downstream, times(1)).onComplete(); + } + + private void mockUpstream() { + Flow.Subscription upstream = mock(); + doAnswer(ans -> { + processor.onNext(output(randomIdentifier())); + return null; + }).when(upstream).request(anyLong()); + processor.onSubscribe(upstream); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java index 1667dac84d2db..ba6bcf8b57d5b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicStreamingProcessorTests.java @@ -11,20 +11,17 @@ import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; -import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; -import org.hamcrest.Matcher; -import org.hamcrest.Matchers; import java.util.ArrayDeque; -import java.util.Arrays; import java.util.Deque; import java.util.Map; import java.util.concurrent.Flow; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onNext; +import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.containsResults; +import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.events; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.isA; import static org.hamcrest.Matchers.notNullValue; @@ -133,21 +130,6 @@ public void testDroppedEventsRequestsMoreData() throws Exception { verify(downstream, times(0)).onNext(any()); } - private Deque events(String... data) { - var item = new ArrayDeque(); - Arrays.stream(data).map(datum -> new ServerSentEvent(ServerSentEventField.DATA, datum)).forEach(item::offer); - return item; - } - - @SuppressWarnings("unchecked") - private Matcher> containsResults(String... results) { - Matcher[] resultMatcher = Arrays.stream(results) - .map(StreamingChatCompletionResults.Result::new) - .map(Matchers::equalTo) - .toArray(Matcher[]::new); - return Matchers.contains(resultMatcher); - } - private static ElasticsearchStatusException onError(Deque item) { var processor = new AnthropicStreamingProcessor(); var response = new AtomicReference(); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessorTests.java new file mode 100644 index 0000000000000..f41fe5b765c8c --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioStreamingProcessorTests.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.googleaistudio; + +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioCompletionResponseEntity; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; + +import java.util.ArrayDeque; +import java.util.concurrent.Flow; + +import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onError; +import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onNext; +import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.containsResults; +import static org.elasticsearch.xpack.inference.external.response.streaming.StreamingInferenceTestUtils.events; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class GoogleAiStudioStreamingProcessorTests extends ESTestCase { + + public void testParseSuccess() { + var item = events(""" + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Hello" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "safetyRatings": [ + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "probability": "NEGLIGIBLE" + } + ] + } + ], + "usageMetadata": { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "totalTokenCount": 1 + } + }""", """ + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": ", World" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "safetyRatings": [ + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "probability": "NEGLIGIBLE" + } + ] + } + ], + "usageMetadata": { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "totalTokenCount": 1 + } + }"""); + + var response = onNext(new GoogleAiStudioStreamingProcessor(GoogleAiStudioCompletionResponseEntity::content), item); + assertThat(response.results().size(), equalTo(2)); + assertThat(response.results(), containsResults("Hello", ", World")); + } + + public void testEmptyResultsRequestsMoreData() throws Exception { + var emptyDeque = new ArrayDeque(); + + var processor = new GoogleAiStudioStreamingProcessor(noOp -> { + fail("This should not be called"); + return null; + }); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(emptyDeque); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testOnError() { + var expectedException = new RuntimeException("hello"); + + var processor = new GoogleAiStudioStreamingProcessor(noOp -> { throw expectedException; }); + + assertThat(onError(processor, events("hi")), sameInstance(expectedException)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java index be47d8806aade..a400b67b3761f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/StreamingHttpResultPublisherTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.junit.Before; +import org.mockito.ArgumentCaptor; import java.io.IOException; import java.nio.ByteBuffer; @@ -38,6 +39,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -59,7 +61,7 @@ public void setUp() throws Exception { super.setUp(); threadPool = mock(ThreadPool.class); settings = mock(HttpSettings.class); - listener = ActionListener.noop(); + listener = spy(ActionListener.noop()); when(threadPool.executor(UTILITY_THREAD_POOL_NAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); when(settings.getMaxResponseSize()).thenReturn(ByteSizeValue.ofBytes(maxBytes)); @@ -235,33 +237,13 @@ public void testTotalBytesDecrement() throws IOException { } /** - * Given an error from Apache - * When the subscriber requests the next set of data - * Then the subscriber receives the error from Apache + * When there is an error from Apache before the publisher invokes the listener + * Then the publisher will forward the call to the listener's onFailure */ public void testErrorBeforeRequest() { - var subscriber = subscribe(); var exception = new NullPointerException("test"); - publisher.failed(exception); - assertThat("subscriber receives exception on next request", subscriber.throwable, nullValue()); - - subscriber.requestData(); - assertThat("subscriber receives exception", subscriber.throwable, is(exception)); - } - - /** - * Given the subscriber is waiting for data - * When Apache sends an error - * Then the subscriber immediately receives the error - */ - public void testErrorAfterRequest() { - var subscriber = subscribe(); - var exception = new NullPointerException("test"); - - subscriber.requestData(); - publisher.failed(exception); - assertThat("subscriber receives exception", subscriber.throwable, is(exception)); + verify(listener).onFailure(exception); } /** @@ -375,6 +357,76 @@ public void testCancelAfterRequest() { assertTrue("onComplete should be called", subscriber.completed); } + /** + * When cancel is called + * Then we only send onComplete once + */ + public void testCancelIsIdempotent() throws IOException { + Flow.Subscriber subscriber = mock(); + + var subscription = ArgumentCaptor.forClass(Flow.Subscription.class); + publisher.subscribe(subscriber); + verify(subscriber).onSubscribe(subscription.capture()); + + publisher.responseReceived(mock()); + publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); + subscription.getValue().request(1); + + subscription.getValue().request(1); + publisher.cancel(); + verify(subscriber, times(1)).onComplete(); + subscription.getValue().request(1); + publisher.cancel(); + verify(subscriber, times(1)).onComplete(); + } + + /** + * When close is called + * Then we only send onComplete once + */ + public void testCloseIsIdempotent() throws IOException { + Flow.Subscriber subscriber = mock(); + + var subscription = ArgumentCaptor.forClass(Flow.Subscription.class); + publisher.subscribe(subscriber); + verify(subscriber).onSubscribe(subscription.capture()); + + publisher.responseReceived(mock()); + publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); + subscription.getValue().request(1); + + subscription.getValue().request(1); + publisher.close(); + verify(subscriber, times(1)).onComplete(); + subscription.getValue().request(1); + publisher.close(); + verify(subscriber, times(1)).onComplete(); + } + + /** + * When failed is called + * Then we only send onError once + */ + public void testFailedIsIdempotent() throws IOException { + var expectedException = new IllegalStateException("wow"); + Flow.Subscriber subscriber = mock(); + + var subscription = ArgumentCaptor.forClass(Flow.Subscription.class); + publisher.subscribe(subscriber); + verify(subscriber).onSubscribe(subscription.capture()); + + publisher.responseReceived(mock()); + publisher.consumeContent(contentDecoder(message), mock(IOControl.class)); + subscription.getValue().request(1); + + subscription.getValue().request(1); + publisher.failed(expectedException); + verify(subscriber, times(1)).onError(eq(expectedException)); + subscription.getValue().request(1); + publisher.failed(expectedException); + verify(subscriber, times(1)).onError(eq(expectedException)); + } + /** * Given the queue is being processed * When Apache cancels the publisher diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java index a57e7c1b64c07..90d0e8742f733 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java @@ -149,6 +149,42 @@ public void testDoneMessageIsIgnored() throws Exception { verify(downstream, times(0)).onNext(any()); } + public void testInitialLlamaResponseIsIgnored() throws Exception { + var item = new ArrayDeque(); + item.offer(new ServerSentEvent(ServerSentEventField.DATA, """ + { + "id":"12345", + "object":"chat.completion.chunk", + "created":123456789, + "model":"Llama-2-7b-chat", + "system_fingerprint": "123456789", + "choices":[ + { + "index":0, + "delta":{ + "role":"assistant" + }, + "logprobs":null, + "finish_reason":null + } + ] + } + """)); + + var processor = new OpenAiStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(item); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + private String toJsonString(ChunkedToXContent chunkedToXContent) throws IOException { try (var builder = XContentFactory.jsonBuilder()) { chunkedToXContent.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xContent -> { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntityTests.java deleted file mode 100644 index 10c8943c75f6c..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAI21LabsCompletionRequestEntityTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import org.elasticsearch.test.ESTestCase; - -import java.util.List; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest; -import static org.hamcrest.Matchers.is; - -public class AmazonBedrockAI21LabsCompletionRequestEntityTests extends ESTestCase { - public void testRequestEntity_CreatesProperRequest() { - var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertThat(builtRequest.modelId(), is("testmodel")); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTemperature() { - var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), 1.0, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopP() { - var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), null, 1.0, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithMaxTokens() { - var request = new AmazonBedrockAI21LabsCompletionRequestEntity(List.of("test message"), null, null, 128); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntityTests.java deleted file mode 100644 index e8a3440a37294..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockAnthropicCompletionRequestEntityTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import org.elasticsearch.test.ESTestCase; - -import java.util.List; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest; -import static org.hamcrest.Matchers.is; - -public class AmazonBedrockAnthropicCompletionRequestEntityTests extends ESTestCase { - public void testRequestEntity_CreatesProperRequest() { - var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertThat(builtRequest.modelId(), is("testmodel")); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTemperature() { - var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), 1.0, null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopP() { - var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, 1.0, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithMaxTokens() { - var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, null, null, 128); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopK() { - var request = new AmazonBedrockAnthropicCompletionRequestEntity(List.of("test message"), null, null, 1.0, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopKInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactoryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactoryTests.java new file mode 100644 index 0000000000000..32cd21bc3d45a --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockChatCompletionEntityFactoryTests.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider; +import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionModel; +import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionServiceSettings; +import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionTaskSettings; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xcontent.XContentParserConfiguration.EMPTY; +import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.AI21LABS; +import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.AMAZONTITAN; +import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.ANTHROPIC; +import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.COHERE; +import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.META; +import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider.MISTRAL; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AmazonBedrockChatCompletionEntityFactoryTests extends ESTestCase { + public void testEntitiesWithoutAdditionalMessages() { + List.of(AI21LABS, AMAZONTITAN, META).forEach(provider -> { + var expectedTemp = randomDoubleBetween(1, 10, true); + var expectedTopP = randomDoubleBetween(1, 10, true); + + var expectedMaxToken = randomIntBetween(1, 10); + var expectedMessage = List.of(randomIdentifier()); + var model = model(provider, expectedTemp, expectedTopP, expectedMaxToken); + + var entity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, expectedMessage); + + assertThat(entity, notNullValue()); + assertThat(entity.temperature(), equalTo(expectedTemp)); + assertThat(entity.topP(), equalTo(expectedTopP)); + assertThat(entity.maxTokenCount(), equalTo(expectedMaxToken)); + assertThat(entity.additionalModelFields(), nullValue()); + assertThat(entity.messages(), equalTo(expectedMessage)); + }); + } + + public void testWithAdditionalMessages() { + List.of(ANTHROPIC, COHERE, MISTRAL).forEach(provider -> { + var expectedTemp = randomDoubleBetween(1, 10, true); + var expectedTopP = randomDoubleBetween(1, 10, true); + var expectedMaxToken = randomIntBetween(1, 10); + var expectedMessage = List.of(randomIdentifier()); + var expectedTopK = randomDoubleBetween(1, 10, true); + var model = model(provider, expectedTemp, expectedTopP, expectedMaxToken, expectedTopK); + + var entity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, expectedMessage); + + assertThat(entity, notNullValue()); + assertThat(entity.temperature(), equalTo(expectedTemp)); + assertThat(entity.topP(), equalTo(expectedTopP)); + assertThat(entity.maxTokenCount(), equalTo(expectedMaxToken)); + assertThat(entity.messages(), equalTo(expectedMessage)); + assertThat(entity.additionalModelFields(), notNullValue()); + assertThat(entity.additionalModelFields().size(), equalTo(1)); + try (var parser = XContentFactory.xContent(XContentType.JSON).createParser(EMPTY, entity.additionalModelFields().getFirst())) { + var additionalModelFields = parser.map(); + assertThat((Double) additionalModelFields.get("top_k"), closeTo(expectedTopK, 0.1)); + } catch (IOException e) { + fail(e); + } + }); + } + + AmazonBedrockChatCompletionModel model(AmazonBedrockProvider provider, Double temperature, Double topP, Integer maxTokenCount) { + return model(provider, temperature, topP, maxTokenCount, null); + } + + AmazonBedrockChatCompletionModel model(AmazonBedrockProvider provider, Double temp, Double topP, Integer tokenCount, Double topK) { + var serviceSettings = mock(AmazonBedrockChatCompletionServiceSettings.class); + when(serviceSettings.provider()).thenReturn(provider); + + var taskSettings = mock(AmazonBedrockChatCompletionTaskSettings.class); + when(taskSettings.temperature()).thenReturn(temp); + when(taskSettings.topP()).thenReturn(topP); + when(taskSettings.maxNewTokens()).thenReturn(tokenCount); + when(taskSettings.topK()).thenReturn(topK); + + var model = mock(AmazonBedrockChatCompletionModel.class); + when(model.getServiceSettings()).thenReturn(serviceSettings); + when(model.getTaskSettings()).thenReturn(taskSettings); + return model; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntityTests.java deleted file mode 100644 index c8e844d000240..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockCohereCompletionRequestEntityTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import org.elasticsearch.test.ESTestCase; - -import java.util.List; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest; -import static org.hamcrest.Matchers.is; - -public class AmazonBedrockCohereCompletionRequestEntityTests extends ESTestCase { - public void testRequestEntity_CreatesProperRequest() { - var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertThat(builtRequest.modelId(), is("testmodel")); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTemperature() { - var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), 1.0, null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopP() { - var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, 1.0, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithMaxTokens() { - var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, null, null, 128); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopK() { - var request = new AmazonBedrockCohereCompletionRequestEntity(List.of("test message"), null, null, 1.0, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopKInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java index 17c3b4488bae4..0e7acd3337e0f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockConverseRequestUtils.java @@ -15,12 +15,16 @@ import java.util.Collection; +import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.getConverseMessageList; +import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseUtils.inferenceConfig; + public final class AmazonBedrockConverseRequestUtils { public static ConverseRequest getConverseRequest(String modelId, AmazonBedrockConverseRequestEntity requestEntity) { - var converseRequest = ConverseRequest.builder().modelId(modelId); - converseRequest = requestEntity.addMessages(converseRequest); - converseRequest = requestEntity.addInferenceConfig(converseRequest); - converseRequest = requestEntity.addAdditionalModelFields(converseRequest); + var converseRequest = ConverseRequest.builder() + .modelId(modelId) + .messages(getConverseMessageList(requestEntity.messages())) + .additionalModelResponseFieldPaths(requestEntity.additionalModelFields()); + inferenceConfig(requestEntity).ifPresent(converseRequest::inferenceConfig); return converseRequest.build(); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntityTests.java deleted file mode 100644 index 25700f7c7aee1..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMetaCompletionRequestEntityTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import org.elasticsearch.test.ESTestCase; - -import java.util.List; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest; -import static org.hamcrest.Matchers.is; - -public class AmazonBedrockMetaCompletionRequestEntityTests extends ESTestCase { - public void testRequestEntity_CreatesProperRequest() { - var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertThat(builtRequest.modelId(), is("testmodel")); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTemperature() { - var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), 1.0, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopP() { - var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), null, 1.0, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithMaxTokens() { - var request = new AmazonBedrockMetaCompletionRequestEntity(List.of("test message"), null, null, 128); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntityTests.java deleted file mode 100644 index 8e321b0cb33a7..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockMistralCompletionRequestEntityTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import org.elasticsearch.test.ESTestCase; - -import java.util.List; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest; -import static org.hamcrest.Matchers.is; - -public class AmazonBedrockMistralCompletionRequestEntityTests extends ESTestCase { - public void testRequestEntity_CreatesProperRequest() { - var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertThat(builtRequest.modelId(), is("testmodel")); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTemperature() { - var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), 1.0, null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopP() { - var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, 1.0, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithMaxTokens() { - var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, null, null, 128); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopK() { - var request = new AmazonBedrockMistralCompletionRequestEntity(List.of("test message"), null, null, 1.0, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopKInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntityTests.java deleted file mode 100644 index 8d1c15499bfb6..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/amazonbedrock/completion/AmazonBedrockTitanCompletionRequestEntityTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion; - -import org.elasticsearch.test.ESTestCase; - -import java.util.List; - -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHasMessage; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopKInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveAnyTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveMaxTokensInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTemperatureInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.doesConverseRequestHaveTopPInput; -import static org.elasticsearch.xpack.inference.external.request.amazonbedrock.completion.AmazonBedrockConverseRequestUtils.getConverseRequest; -import static org.hamcrest.Matchers.is; - -public class AmazonBedrockTitanCompletionRequestEntityTests extends ESTestCase { - public void testRequestEntity_CreatesProperRequest() { - var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), null, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertThat(builtRequest.modelId(), is("testmodel")); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTemperature() { - var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), 1.0, null, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertTrue(doesConverseRequestHaveTemperatureInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithTopP() { - var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), null, 1.0, null); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertTrue(doesConverseRequestHaveTopPInput(builtRequest, 1.0)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyMaxTokensInput(builtRequest)); - } - - public void testRequestEntity_CreatesProperRequest_WithMaxTokens() { - var request = new AmazonBedrockTitanCompletionRequestEntity(List.of("test message"), null, null, 128); - var builtRequest = getConverseRequest("testmodel", request); - assertThat(builtRequest.modelId(), is("testmodel")); - assertThat(doesConverseRequestHasMessage(builtRequest, "test message"), is(true)); - assertFalse(doesConverseRequestHaveAnyTemperatureInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopPInput(builtRequest)); - assertFalse(doesConverseRequestHaveAnyTopKInput(builtRequest)); - assertTrue(doesConverseRequestHaveMaxTokensInput(builtRequest, 128)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java index 7d7ee1dcba6c2..7ffa8940ad6be 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java @@ -10,6 +10,7 @@ import org.apache.http.client.methods.HttpPost; import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioCompletionRequest; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModelTests; @@ -29,7 +30,7 @@ public void testCreateRequest() throws IOException { var apiKey = "api_key"; var input = "input"; - var request = new GoogleAiStudioCompletionRequest(List.of(input), GoogleAiStudioCompletionModelTests.createModel("model", apiKey)); + var request = new GoogleAiStudioCompletionRequest(listOf(input), GoogleAiStudioCompletionModelTests.createModel("model", apiKey)); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); @@ -54,7 +55,7 @@ public void testCreateRequest() throws IOException { public void testTruncate_ReturnsSameInstance() { var request = new GoogleAiStudioCompletionRequest( - List.of("input"), + listOf("input"), GoogleAiStudioCompletionModelTests.createModel("model", "api key") ); var truncatedRequest = request.truncate(); @@ -64,10 +65,14 @@ public void testTruncate_ReturnsSameInstance() { public void testTruncationInfo_ReturnsNull() { var request = new GoogleAiStudioCompletionRequest( - List.of("input"), + listOf("input"), GoogleAiStudioCompletionModelTests.createModel("model", "api key") ); assertNull(request.getTruncationInfo()); } + + private static DocumentsOnlyInput listOf(String... input) { + return new DocumentsOnlyInput(List.of(input)); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/StreamingInferenceTestUtils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/StreamingInferenceTestUtils.java new file mode 100644 index 0000000000000..e0aef58c4f3b3 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/StreamingInferenceTestUtils.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.streaming; + +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; + +public class StreamingInferenceTestUtils { + + public static Deque events(String... data) { + var item = new ArrayDeque(); + Arrays.stream(data).map(datum -> new ServerSentEvent(ServerSentEventField.DATA, datum)).forEach(item::offer); + return item; + } + + @SuppressWarnings("unchecked") + public static Matcher> containsResults(String... results) { + Matcher[] resultMatcher = Arrays.stream(results) + .map(StreamingChatCompletionResults.Result::new) + .map(Matchers::equalTo) + .toArray(Matcher[]::new); + return Matchers.contains(resultMatcher); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java index 75c370fd4d3fb..409d62426949c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.search.SearchHit; @@ -35,16 +36,16 @@ import org.junit.Before; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Map; import java.util.concurrent.TimeUnit; import static org.elasticsearch.core.Strings.format; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.sameInstance; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -292,58 +293,30 @@ public void testStoreModel_ThrowsException_WhenFailureIsNotAVersionConflict() { ); } - @SuppressWarnings("unchecked") - public void testDeepCopyDefaultConfig() { - { - var toCopy = new UnparsedModel("tocopy", randomFrom(TaskType.values()), "service-a", Map.of(), Map.of()); - var copied = ModelRegistry.deepCopyDefaultConfig(toCopy); - assertThat(copied, not(sameInstance(toCopy))); - assertThat(copied.taskType(), is(toCopy.taskType())); - assertThat(copied.service(), is(toCopy.service())); - assertThat(copied.secrets(), not(sameInstance(toCopy.secrets()))); - assertThat(copied.secrets(), is(toCopy.secrets())); - // Test copied is a modifiable map - copied.secrets().put("foo", "bar"); - - assertThat(copied.settings(), not(sameInstance(toCopy.settings()))); - assertThat(copied.settings(), is(toCopy.settings())); - // Test copied is a modifiable map - copied.settings().put("foo", "bar"); - } + public void testIdMatchedDefault() { + var defaultConfigIds = new ArrayList(); + defaultConfigIds.add(new InferenceService.DefaultConfigId("foo", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class))); + defaultConfigIds.add(new InferenceService.DefaultConfigId("bar", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class))); - { - Map secretsMap = Map.of("secret", "value"); - Map chunking = Map.of("strategy", "word"); - Map task = Map.of("user", "name"); - Map service = Map.of("num_threads", 1, "adaptive_allocations", Map.of("enabled", true)); - Map settings = Map.of("chunking_settings", chunking, "service_settings", service, "task_settings", task); - - var toCopy = new UnparsedModel("tocopy", randomFrom(TaskType.values()), "service-a", settings, secretsMap); - var copied = ModelRegistry.deepCopyDefaultConfig(toCopy); - assertThat(copied, not(sameInstance(toCopy))); - - assertThat(copied.secrets(), not(sameInstance(toCopy.secrets()))); - assertThat(copied.secrets(), is(toCopy.secrets())); - // Test copied is a modifiable map - copied.secrets().remove("secret"); - - assertThat(copied.settings(), not(sameInstance(toCopy.settings()))); - assertThat(copied.settings(), is(toCopy.settings())); - // Test copied is a modifiable map - var chunkOut = (Map) copied.settings().get("chunking_settings"); - assertThat(chunkOut, is(chunking)); - chunkOut.remove("strategy"); - - var taskOut = (Map) copied.settings().get("task_settings"); - assertThat(taskOut, is(task)); - taskOut.remove("user"); - - var serviceOut = (Map) copied.settings().get("service_settings"); - assertThat(serviceOut, is(service)); - var adaptiveOut = (Map) serviceOut.remove("adaptive_allocations"); - assertThat(adaptiveOut, is(Map.of("enabled", true))); - adaptiveOut.remove("enabled"); - } + var matched = ModelRegistry.idMatchedDefault("bar", defaultConfigIds); + assertEquals(defaultConfigIds.get(1), matched.get()); + matched = ModelRegistry.idMatchedDefault("baz", defaultConfigIds); + assertFalse(matched.isPresent()); + } + + public void testTaskTypeMatchedDefaults() { + var defaultConfigIds = new ArrayList(); + defaultConfigIds.add(new InferenceService.DefaultConfigId("s1", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class))); + defaultConfigIds.add(new InferenceService.DefaultConfigId("s2", TaskType.SPARSE_EMBEDDING, mock(InferenceService.class))); + defaultConfigIds.add(new InferenceService.DefaultConfigId("d1", TaskType.TEXT_EMBEDDING, mock(InferenceService.class))); + defaultConfigIds.add(new InferenceService.DefaultConfigId("c1", TaskType.COMPLETION, mock(InferenceService.class))); + + var matched = ModelRegistry.taskTypeMatchedDefaults(TaskType.SPARSE_EMBEDDING, defaultConfigIds); + assertThat(matched, contains(defaultConfigIds.get(0), defaultConfigIds.get(1))); + matched = ModelRegistry.taskTypeMatchedDefaults(TaskType.TEXT_EMBEDDING, defaultConfigIds); + assertThat(matched, contains(defaultConfigIds.get(2))); + matched = ModelRegistry.taskTypeMatchedDefaults(TaskType.RERANK, defaultConfigIds); + assertThat(matched, empty()); } public void testDuplicateDefaultIds() { @@ -351,11 +324,15 @@ public void testDuplicateDefaultIds() { var registry = new ModelRegistry(client); var id = "my-inference"; + var mockServiceA = mock(InferenceService.class); + when(mockServiceA.name()).thenReturn("service-a"); + var mockServiceB = mock(InferenceService.class); + when(mockServiceB.name()).thenReturn("service-b"); - registry.addDefaultConfiguration(new UnparsedModel(id, randomFrom(TaskType.values()), "service-a", Map.of(), Map.of())); + registry.addDefaultIds(new InferenceService.DefaultConfigId(id, randomFrom(TaskType.values()), mockServiceA)); var ise = expectThrows( IllegalStateException.class, - () -> registry.addDefaultConfiguration(new UnparsedModel(id, randomFrom(TaskType.values()), "service-b", Map.of(), Map.of())) + () -> registry.addDefaultIds(new InferenceService.DefaultConfigId(id, randomFrom(TaskType.values()), mockServiceB)) ); assertThat( ise.getMessage(), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index 9c746e7c2aed9..06c5a68987a9e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.inference.services.amazonbedrock; +import software.amazon.awssdk.services.bedrockruntime.model.BedrockRuntimeException; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; @@ -33,7 +35,6 @@ import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.Utils; import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockMockRequestSender; -import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponentsTests; @@ -1265,12 +1266,19 @@ public void testInfer_UnauthorizedResponse() throws IOException { var factory = mock(HttpRequestSender.Factory.class); when(factory.createSender()).thenReturn(sender); - var amazonBedrockFactory = new AmazonBedrockRequestSender.Factory( + var amazonBedrockFactory = new AmazonBedrockMockRequestSender.Factory( ServiceComponentsTests.createWithSettings(threadPool, Settings.EMPTY), mockClusterServiceEmpty() ); - try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { + try ( + var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool)); + var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender() + ) { + requestSender.enqueue( + BedrockRuntimeException.builder().message("The security token included in the request is invalid").build() + ); + var model = AmazonBedrockEmbeddingsModelTests.createModel( "id", "us-east-1", diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index 61645613b8722..860642a23fb2c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -14,7 +14,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -38,6 +40,7 @@ import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; @@ -171,7 +174,7 @@ public void testParseRequestConfig_Misconfigured() { public void testParseRequestConfig_E5() { { - var service = createService(mock(Client.class), Set.of("Aarch64")); + var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC); var settings = new HashMap(); settings.put( ModelConfigurations.SERVICE_SETTINGS, @@ -198,7 +201,7 @@ public void testParseRequestConfig_E5() { } { - var service = createService(mock(Client.class), Set.of("linux-x86_64")); + var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.LINUX_X86_OPTIMIZED); var settings = new HashMap(); settings.put( ModelConfigurations.SERVICE_SETTINGS, @@ -231,7 +234,7 @@ public void testParseRequestConfig_E5() { // Invalid service settings { - var service = createService(mock(Client.class), Set.of("Aarch64")); + var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC); var settings = new HashMap(); settings.put( ModelConfigurations.SERVICE_SETTINGS, @@ -267,7 +270,7 @@ public void testParseRequestConfig_E5() { } ); - var service = createService(mock(Client.class), Set.of("Aarch64")); + var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC); var settings = new HashMap(); settings.put( ModelConfigurations.SERVICE_SETTINGS, @@ -289,7 +292,7 @@ public void testParseRequestConfig_E5() { { assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); - var service = createService(mock(Client.class), Set.of("Aarch64")); + var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC); var settings = new HashMap(); settings.put( ModelConfigurations.SERVICE_SETTINGS, @@ -318,7 +321,7 @@ public void testParseRequestConfig_E5() { { assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); - var service = createService(mock(Client.class), Set.of("Aarch64")); + var service = createService(mock(Client.class), BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC); var settings = new HashMap(); settings.put( ModelConfigurations.SERVICE_SETTINGS, @@ -1492,26 +1495,33 @@ public void testParseRequestConfigEland_SetsDimensionsToOne() { public void testModelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic() { { - var architectures = Set.of("Aarch64"); assertFalse( - ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86) + ElasticsearchInternalService.modelVariantValidForArchitecture( + BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC, + MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 + ) ); - assertTrue(ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID)); - } - { - var architectures = Set.of("linux-x86_64"); assertTrue( - ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86) + ElasticsearchInternalService.modelVariantValidForArchitecture( + BaseElasticsearchInternalService.PreferredModelVariant.PLATFORM_AGNOSTIC, + MULTILINGUAL_E5_SMALL_MODEL_ID + ) ); - assertTrue(ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID)); } { - var architectures = Set.of("linux-x86_64", "Aarch64"); - assertFalse( - ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86) + assertTrue( + ElasticsearchInternalService.modelVariantValidForArchitecture( + BaseElasticsearchInternalService.PreferredModelVariant.LINUX_X86_OPTIMIZED, + MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 + ) + ); + assertTrue( + ElasticsearchInternalService.modelVariantValidForArchitecture( + BaseElasticsearchInternalService.PreferredModelVariant.LINUX_X86_OPTIMIZED, + MULTILINGUAL_E5_SMALL_MODEL_ID + ) ); - assertTrue(ElasticsearchInternalService.modelVariantValidForArchitecture(architectures, MULTILINGUAL_E5_SMALL_MODEL_ID)); } } @@ -1541,13 +1551,28 @@ public void testEmbeddingTypeFromTaskTypeAndSettings() { assertThat(e.getMessage(), containsString("Chunking is not supported for task type [completion]")); } + public void testIsDefaultId() { + var service = createService(mock(Client.class)); + assertTrue(service.isDefaultId(".elser-2")); + assertTrue(service.isDefaultId(".multi-e5-small")); + assertFalse(service.isDefaultId("foo")); + } + private ElasticsearchInternalService createService(Client client) { - var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool); + var cs = mock(ClusterService.class); + var cSettings = new ClusterSettings(Settings.EMPTY, Set.of(MachineLearningField.MAX_LAZY_ML_NODES)); + when(cs.getClusterSettings()).thenReturn(cSettings); + var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool, cs, Settings.EMPTY); return new ElasticsearchInternalService(context); } - private ElasticsearchInternalService createService(Client client, Set architectures) { - var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool); - return new ElasticsearchInternalService(context, l -> l.onResponse(architectures)); + private ElasticsearchInternalService createService(Client client, BaseElasticsearchInternalService.PreferredModelVariant modelVariant) { + var context = new InferenceServiceExtension.InferenceServiceFactoryContext( + client, + threadPool, + mock(ClusterService.class), + Settings.EMPTY + ); + return new ElasticsearchInternalService(context, l -> l.onResponse(modelVariant)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java new file mode 100644 index 0000000000000..74cdab79fe79b --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; + +public class ElserInternalModelTests extends ESTestCase { + public void testUpdateNumAllocation() { + var model = new ElserInternalModel( + "foo", + TaskType.SPARSE_EMBEDDING, + ElasticsearchInternalService.NAME, + new ElserInternalServiceSettings(null, 1, "elser", null), + new ElserMlNodeTaskSettings(), + null + ); + + model.updateNumAllocation(1); + assertEquals(1, model.internalServiceSettings.getNumAllocations().intValue()); + + model.updateNumAllocation(null); + assertNull(model.internalServiceSettings.getNumAllocations()); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java index f4c13db78c4bc..3d523d7cab498 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java @@ -14,11 +14,15 @@ import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import java.net.URI; import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; public class GoogleAiStudioCompletionModelTests extends ESTestCase { @@ -55,14 +59,17 @@ public static GoogleAiStudioCompletionModel createModel(String model, String api } public static GoogleAiStudioCompletionModel createModel(String model, String url, String apiKey) { - return new GoogleAiStudioCompletionModel( - "id", - TaskType.COMPLETION, - "service", - url, - new GoogleAiStudioCompletionServiceSettings(model, null), - EmptyTaskSettings.INSTANCE, - new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + var googleModel = spy( + new GoogleAiStudioCompletionModel( + "id", + TaskType.COMPLETION, + "service", + new GoogleAiStudioCompletionServiceSettings(model, null), + EmptyTaskSettings.INSTANCE, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ) ); + when(googleModel.uri(anyBoolean())).thenReturn(URI.create(url)); + return googleModel; } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index b463426de0848..ee9d6129dcd54 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -42,7 +42,7 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { public Settings getAdditionalIndexSettings( final String indexName, final String dataStreamName, - boolean isTimeSeries, + IndexMode templateIndexMode, final Metadata metadata, final Instant resolvedAt, final Settings settings, diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java index 759fa6af98868..a190ff72de8df 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java @@ -50,7 +50,7 @@ final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider public Settings getAdditionalIndexSettings( String indexName, String dataStreamName, - boolean isTimeSeries, + IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, @@ -59,7 +59,7 @@ public Settings getAdditionalIndexSettings( // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) boolean isTemplateValidation = "validate-index-name".equals(indexName); - if (newIndexHasSyntheticSourceUsage(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings, combinedTemplateMappings) + if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings, combinedTemplateMappings) && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation)) { LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName); // TODO: handle falling back to stored source @@ -69,7 +69,7 @@ public Settings getAdditionalIndexSettings( boolean newIndexHasSyntheticSourceUsage( String indexName, - boolean isTimeSeries, + IndexMode templateIndexMode, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings ) { @@ -79,15 +79,17 @@ boolean newIndexHasSyntheticSourceUsage( return false; } - var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings); - try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) { - // combinedTemplateMappings can be null when creating system indices - // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping. - if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) { - combinedTemplateMappings = List.of(new CompressedXContent("{}")); + try { + var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); + try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) { + // combinedTemplateMappings can be null when creating system indices + // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping. + if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) { + combinedTemplateMappings = List.of(new CompressedXContent("{}")); + } + mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); + return mapperService.documentMapper().sourceMapper().isSynthetic(); } - mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - return mapperService.documentMapper().sourceMapper().isSynthetic(); } catch (AssertionError | Exception e) { // In case invalid mappings or setting are provided, then mapper service creation can fail. // In that case it is ok to return false here. The index creation will fail anyway later, so need to fallback to stored source. @@ -99,7 +101,7 @@ boolean newIndexHasSyntheticSourceUsage( // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used: private IndexMetadata buildIndexMetadataForMapperService( String indexName, - boolean isTimeSeries, + IndexMode templateIndexMode, Settings indexTemplateAndCreateRequestSettings ) { var tmpIndexMetadata = IndexMetadata.builder(indexName); @@ -117,7 +119,7 @@ private IndexMetadata buildIndexMetadataForMapperService( .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - if (isTimeSeries) { + if (templateIndexMode == IndexMode.TIME_SERIES) { finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); // Avoid failing because index.routing_path is missing (in case fields are marked as dimension) finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path")); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 04e89af254f64..5f23dbdca1143 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -51,7 +51,7 @@ public void testLogsDbDisabled() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -69,7 +69,7 @@ public void testOnIndexCreation() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( "logs-apache-production", null, - false, + null, Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -87,7 +87,7 @@ public void testOnExplicitStandardIndex() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), @@ -105,7 +105,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), @@ -123,7 +123,7 @@ public void testNonLogsDataStream() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs", - false, + null, Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -141,7 +141,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("*"), List.of()), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -159,7 +159,7 @@ public void testWithLogsComponentTemplate() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -177,7 +177,7 @@ public void testWithMultipleComponentTemplates() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -195,7 +195,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -213,7 +213,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -231,7 +231,7 @@ public void testCaseSensitivity() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "LOGS-apache-production", - false, + null, Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -249,7 +249,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production-eu", - false, + null, Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -267,7 +267,7 @@ public void testBeforeAndAFterSettingUpdate() throws IOException { final Settings beforeSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -281,7 +281,7 @@ public void testBeforeAndAFterSettingUpdate() throws IOException { final Settings afterSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, @@ -295,7 +295,7 @@ public void testBeforeAndAFterSettingUpdate() throws IOException { final Settings laterSettings = provider.getAdditionalIndexSettings( null, "logs-apache-production", - false, + null, buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java index c97328da132bd..738487b9365a7 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java @@ -49,7 +49,7 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } } """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertTrue(result); } { @@ -82,7 +82,7 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } """; } - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertFalse(result); } } @@ -104,7 +104,7 @@ public void testValidateIndexName() throws IOException { } """; Settings settings = Settings.EMPTY; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertFalse(result); } @@ -124,22 +124,22 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException """; { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertTrue(result); } { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of()); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); assertTrue(result); } { - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, Settings.EMPTY, List.of()); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); assertFalse(result); } { boolean result = provider.newIndexHasSyntheticSourceUsage( indexName, - false, + null, Settings.EMPTY, List.of(new CompressedXContent(mapping)) ); @@ -164,22 +164,22 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { """; { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertTrue(result); } { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of()); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); assertTrue(result); } { - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, Settings.EMPTY, List.of()); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); assertFalse(result); } { boolean result = provider.newIndexHasSyntheticSourceUsage( indexName, - false, + null, Settings.EMPTY, List.of(new CompressedXContent(mapping)) ); @@ -206,7 +206,7 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep } } """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertFalse(result); } { @@ -221,7 +221,7 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep } } """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, false, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertFalse(result); } } diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml new file mode 100644 index 0000000000000..33fedce3b59c1 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml @@ -0,0 +1,847 @@ +--- +create an index with disabled source mode and standard index mode without setting: + - do: + indices.create: + index: test_disabled_standard + body: + settings: + index: + mode: standard + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_standard + + - match: { test_disabled_standard.mappings._source.mode: disabled } + +--- +create an index with stored source mode and standard index mode without setting: + - do: + indices.create: + index: test_stored_standard + body: + settings: + index: + mode: standard + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_standard + + - match: { test_stored_standard.mappings._source.mode: stored } + +--- +create an index with synthetic source mode and standard index mode without setting: + - do: + indices.create: + index: test_synthetic_standard + body: + settings: + index: + mode: standard + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_standard + + - match: { test_synthetic_standard.mappings._source.mode: synthetic } + +--- +create an index with disabled source mode and logsdb index mode without setting: + - do: + catch: bad_request + indices.create: + index: test_disabled_logsdb + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: disabled + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +create an index with stored source mode and logsdb index mode without setting: + - do: + indices.create: + index: test_stored_logsdb + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: stored + + - do: + indices.get_settings: + index: "test_stored_logsdb" + - match: { test_stored_logsdb.settings.index.mode: logsdb } + + - do: + indices.get_mapping: + index: test_stored_logsdb + + - match: { test_stored_logsdb.mappings._source.mode: stored } + +--- +create an index with synthetic source mode and logsdb index mode without setting: + - do: + indices.create: + index: test_synthetic_logsdb + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_logsdb + + - match: { test_synthetic_logsdb.mappings._source.mode: synthetic } + +--- +create an index with disabled source mode and time series index mode without setting: + - do: + catch: bad_request + indices.create: + index: test_disabled_time_series + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + _source: + mode: disabled + properties: + keyword: + type: keyword + time_series_dimension: true + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + +--- +create an index with stored source mode and time series index mode without setting: + - do: + indices.create: + index: test_stored_time_series + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + _source: + mode: stored + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_stored_time_series" + - match: { test_stored_time_series.settings.index.mode: time_series } + + - do: + indices.get_mapping: + index: test_stored_time_series + + - match: { test_stored_time_series.mappings._source.mode: stored } + + +--- +create an index with synthetic source mode and time series index mode without setting: + - do: + indices.create: + index: test_synthetic_time_series + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + _source: + mode: synthetic + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_synthetic_time_series" + - match: { test_synthetic_time_series.settings.index.mode: time_series } + + - do: + indices.get_mapping: + index: test_synthetic_time_series + + - match: { test_synthetic_time_series.mappings._source.mode: synthetic } + +--- +create an index with stored source mode: + - do: + indices.create: + index: test_stored_default + body: + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_default + + - match: { test_stored_default.mappings._source.mode: stored } + +--- +override stored to synthetic source mode: + - do: + indices.create: + index: test_stored_override + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_override + + - match: { test_stored_override.mappings._source.mode: synthetic } + +--- +override stored to disabled source mode: + - do: + indices.create: + index: test_stored_disabled + body: + settings: + index: + mapping.source.mode: disabled + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_disabled + + - match: { test_stored_disabled.mappings._source.mode: disabled } + +--- +create an index with disabled source mode: + - do: + indices.create: + index: test_disabled_default + body: + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_default + + - match: { test_disabled_default.mappings._source.mode: disabled } + +--- +override disabled to synthetic source mode: + - do: + indices.create: + index: test_disabled_synthetic + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_synthetic + + - match: { test_disabled_synthetic.mappings._source.mode: synthetic } + +--- +override disabled to stored source mode: + - do: + indices.create: + index: test_disabled_stored + body: + settings: + index: + mapping.source.mode: stored + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_stored + + - match: { test_disabled_stored.mappings._source.mode: stored } + +--- +create an index with synthetic source mode: + - do: + indices.create: + index: test_synthetic_default + body: + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_default + + - match: { test_synthetic_default.mappings._source.mode: synthetic } + +--- +override synthetic to stored source mode: + - do: + indices.create: + index: test_synthetic_stored + body: + settings: + index: + mapping.source.mode: stored + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_stored + + - match: { test_synthetic_stored.mappings._source.mode: stored } + +--- +override synthetic to disabled source mode: + - do: + indices.create: + index: test_synthetic_disabled + body: + settings: + index: + mapping.source.mode: disabled + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_disabled + + - match: { test_synthetic_disabled.mappings._source.mode: disabled } + +--- +create an index with unspecified source mode: + - do: + indices.create: + index: test_unset_default + + - do: + indices.get_mapping: + index: test_unset_default + + - match: { test_unset_default.mappings._source.mode: null } + +--- +override unspecified to stored source mode: + - do: + indices.create: + index: test_unset_stored + body: + settings: + index: + mapping.source.mode: stored + + - do: + indices.get_mapping: + index: test_unset_stored + + - match: { test_unset_stored.mappings: { } } + +--- +override unspecified to disabled source mode: + - do: + indices.create: + index: test_unset_disabled + body: + settings: + index: + mapping.source.mode: disabled + + - do: + indices.get_mapping: + index: test_unset_disabled + + - match: { test_unset_disabled.mappings: { } } + +--- +override unspecified to synthetic source mode: + - do: + indices.create: + index: test_unset_synthetic + body: + settings: + index: + mapping.source.mode: synthetic + + - do: + indices.get_mapping: + index: test_unset_synthetic + + - match: { test_unset_synthetic.mappings: { } } + +--- +create an index with standard index mode: + - do: + indices.create: + index: test_standard_index_mode + body: + settings: + index: + mode: standard + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_standard_index_mode + + - match: { test_standard_index_mode.mappings._source.mode: stored } + +--- +create an index with time_series index mode and synthetic source: + - do: + indices.create: + index: test_time_series_index_mode_synthetic + body: + settings: + index: + mode: time_series + mapping.source.mode: synthetic + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_time_series_index_mode_synthetic" + - match: { test_time_series_index_mode_synthetic.settings.index.mode: time_series } + + + - do: + indices.get_mapping: + index: test_time_series_index_mode_synthetic + + - match: { test_time_series_index_mode_synthetic.mappings._source.mode: synthetic } + +--- +create an index with logsdb index mode and synthetic source: + - do: + indices.create: + index: test_logsdb_index_mode_synthetic + body: + settings: + index: + mode: logsdb + mapping.source.mode: synthetic + + - do: + indices.get_settings: + index: "test_logsdb_index_mode_synthetic" + - match: { test_logsdb_index_mode_synthetic.settings.index.mode: logsdb } + + - do: + indices.get_mapping: + index: test_logsdb_index_mode_synthetic + + - match: { test_logsdb_index_mode_synthetic.mappings._source.mode: synthetic } + +--- +create an index with time_series index mode and stored source: + - do: + indices.create: + index: test_time_series_index_mode_undefined + body: + settings: + index: + mode: time_series + mapping.source.mode: stored + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_time_series_index_mode_undefined" + - match: { test_time_series_index_mode_undefined.settings.index.mode: time_series } + + - do: + indices.get_mapping: + index: test_time_series_index_mode_undefined + + - match: { test_time_series_index_mode_undefined.mappings._source.mode: stored } + +--- +create an index with logsdb index mode and stored source: + - do: + indices.create: + index: test_logsdb_index_mode_undefined + body: + settings: + index: + mode: logsdb + mapping.source.mode: stored + + - do: + indices.get_settings: + index: "test_logsdb_index_mode_undefined" + - match: { test_logsdb_index_mode_undefined.settings.index.mode: logsdb } + + - do: + indices.get_mapping: + index: test_logsdb_index_mode_undefined + + - match: { test_logsdb_index_mode_undefined.mappings._source.mode: stored } + +--- +create an index with time_series index mode and disabled source: + - do: + catch: bad_request + indices.create: + index: test_time_series_index_mode + body: + settings: + index: + mode: time_series + mapping.source.mode: disabled + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + +--- +create an index with logsdb index mode and disabled source: + - do: + catch: bad_request + indices.create: + index: test_logsdb_index_mode + body: + settings: + index: + mode: logsdb + mapping.source.mode: disabled + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +modify final setting after index creation: + - do: + indices.create: + index: test_modify_setting + body: + settings: + index: + mapping.source.mode: stored + + - do: + catch: /.*Can't update non dynamic setting.*/ + indices.put_settings: + index: test_modify_setting + body: + index: + mapping.source.mode: synthetic + +--- +modify source mapping from stored to disabled after index creation: + - do: + indices.create: + index: test_modify_source_mode_stored_disabled + body: + settings: + index: + mapping.source.mode: stored + + - do: + indices.put_mapping: + index: test_modify_source_mode_stored_disabled + body: + _source: + mode: disabled + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_stored_disabled + - match: { test_modify_source_mode_stored_disabled.mappings._source.mode: stored } + +--- +modify source mapping from stored to synthetic after index creation: + - do: + indices.create: + index: test_modify_source_mode_stored_synthetic + body: + settings: + index: + mapping.source.mode: stored + + - do: + indices.put_mapping: + index: test_modify_source_mode_stored_synthetic + body: + _source: + mode: synthetic + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_stored_synthetic + - match: { test_modify_source_mode_stored_synthetic.mappings._source.mode: stored } + +--- +modify source mapping from disabled to stored after index creation: + - do: + indices.create: + index: test_modify_source_mode_disabled_stored + body: + settings: + index: + mapping.source.mode: disabled + + - do: + indices.put_mapping: + index: test_modify_source_mode_disabled_stored + body: + _source: + mode: stored + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_disabled_stored + - match: { test_modify_source_mode_disabled_stored.mappings._source.mode: disabled } + +--- +modify source mapping from disabled to synthetic after index creation: + - do: + indices.create: + index: test_modify_source_mode_disabled_synthetic + body: + settings: + index: + mapping.source.mode: disabled + + - do: + indices.put_mapping: + index: test_modify_source_mode_disabled_synthetic + body: + _source: + mode: synthetic + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_disabled_synthetic + - match: { test_modify_source_mode_disabled_synthetic.mappings._source.mode: disabled } + +--- +modify source mapping from synthetic to stored after index creation: + - do: + indices.create: + index: test_modify_source_mode_synthetic_stored + body: + settings: + index: + mapping.source.mode: synthetic + + - do: + indices.put_mapping: + index: test_modify_source_mode_synthetic_stored + body: + _source: + mode: stored + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_synthetic_stored + - match: { test_modify_source_mode_synthetic_stored.mappings._source.mode: synthetic } + +--- +modify source mapping from synthetic to disabled after index creation: + - do: + indices.create: + index: test_modify_source_mode_synthetic_disabled + body: + settings: + index: + mapping.source.mode: synthetic + + - do: + indices.put_mapping: + index: test_modify_source_mode_synthetic_disabled + body: + _source: + mode: disabled + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_synthetic_disabled + - match: { test_modify_source_mode_synthetic_disabled.mappings._source.mode: synthetic } + +--- +modify logsdb index source mode to disabled after index creation: + - do: + indices.create: + index: test_modify_logsdb_disabled_after_creation + body: + settings: + index: + mode: logsdb + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_logsdb_disabled_after_creation + body: + _source: + mode: disabled + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +modify logsdb index source mode to stored after index creation: + - do: + indices.create: + index: test_modify_logsdb_stored_after_creation + body: + settings: + index: + mode: logsdb + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_logsdb_stored_after_creation + body: + _source: + mode: stored + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" } + +--- +modify time_series index source mode to disabled after index creation: + - do: + indices.create: + index: test_modify_time_series_disabled_after_creation + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_time_series_disabled_after_creation + body: + _source: + mode: disabled + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + +--- +modify time_series index source mode to stored after index creation: + - do: + indices.create: + index: test_modify_time_series_stored_after_creation + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_time_series_stored_after_creation + body: + _source: + mode: stored + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index 51e39f144e44c..554cd0489ae8a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -18,6 +18,7 @@ dependencies { javaRestTestImplementation project(path: xpackModule('esql-core')) javaRestTestImplementation project(path: xpackModule('esql')) javaRestTestImplementation project(path: xpackModule('snapshot-repo-test-kit')) + javaRestTestImplementation project(path: xpackModule('ent-search')) } // location for keys and certificates diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AdaptiveAllocationsScaleFromZeroIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AdaptiveAllocationsScaleFromZeroIT.java new file mode 100644 index 0000000000000..db34ec5eedc86 --- /dev/null +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AdaptiveAllocationsScaleFromZeroIT.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.integration; + +import org.apache.lucene.tests.util.LuceneTestCase; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +@LuceneTestCase.AwaitsFix(bugUrl = "Cannot test without setting the scale to zero period to a small value") +public class AdaptiveAllocationsScaleFromZeroIT extends PyTorchModelRestTestCase { + + @SuppressWarnings("unchecked") + public void testScaleFromZero() throws Exception { + String modelId = "test_scale_from_zero"; + createPassThroughModel(modelId); + putModelDefinition(modelId, PyTorchModelIT.BASE_64_ENCODED_MODEL, PyTorchModelIT.RAW_MODEL_SIZE); + putVocabulary(List.of("Auto", "scale", "and", "infer"), modelId); + + startDeployment(modelId, modelId, new AdaptiveAllocationsSettings(true, 0, 1)); + { + var responseMap = entityAsMap(getTrainedModelStats(modelId)); + List> stats = (List>) responseMap.get("trained_model_stats"); + String statusState = (String) XContentMapValues.extractValue("deployment_stats.allocation_status.state", stats.get(0)); + assertThat(responseMap.toString(), statusState, is(not(nullValue()))); + Integer count = (Integer) XContentMapValues.extractValue("deployment_stats.allocation_status.allocation_count", stats.get(0)); + assertThat(responseMap.toString(), count, is(1)); + } + + // wait for scale down. The scaler service will check every 10 seconds + assertBusy(() -> { + var statsMap = entityAsMap(getTrainedModelStats(modelId)); + List> innerStats = (List>) statsMap.get("trained_model_stats"); + Integer innerCount = (Integer) XContentMapValues.extractValue( + "deployment_stats.allocation_status.allocation_count", + innerStats.get(0) + ); + assertThat(statsMap.toString(), innerCount, is(0)); + }, 30, TimeUnit.SECONDS); + + var failures = new ConcurrentLinkedDeque(); + + // infer will scale up + int inferenceCount = 10; + var latch = new CountDownLatch(inferenceCount); + for (int i = 0; i < inferenceCount; i++) { + asyncInfer("Auto scale and infer", modelId, TimeValue.timeValueSeconds(5), new ResponseListener() { + @Override + public void onSuccess(Response response) { + latch.countDown(); + } + + @Override + public void onFailure(Exception exception) { + latch.countDown(); + failures.add(exception); + } + }); + } + + latch.await(); + assertThat(failures, empty()); + } + + @SuppressWarnings("unchecked") + public void testMultipleDeploymentsWaiting() throws Exception { + String id1 = "test_scale_from_zero_dep_1"; + String id2 = "test_scale_from_zero_dep_2"; + String id3 = "test_scale_from_zero_dep_3"; + var idsList = Arrays.asList(id1, id2, id3); + for (var modelId : idsList) { + createPassThroughModel(modelId); + putModelDefinition(modelId, PyTorchModelIT.BASE_64_ENCODED_MODEL, PyTorchModelIT.RAW_MODEL_SIZE); + putVocabulary(List.of("Auto", "scale", "and", "infer"), modelId); + + startDeployment(modelId, modelId, new AdaptiveAllocationsSettings(true, 0, 1)); + } + + // wait for scale down. The scaler service will check every 10 seconds + assertBusy(() -> { + var statsMap = entityAsMap(getTrainedModelStats("test_scale_from_zero_dep_*")); + List> innerStats = (List>) statsMap.get("trained_model_stats"); + assertThat(innerStats, hasSize(3)); + for (int i = 0; i < 3; i++) { + Integer innerCount = (Integer) XContentMapValues.extractValue( + "deployment_stats.allocation_status.allocation_count", + innerStats.get(i) + ); + assertThat(statsMap.toString(), innerCount, is(0)); + } + }, 30, TimeUnit.SECONDS); + + // infer will scale up + int inferenceCount = 10; + var latch = new CountDownLatch(inferenceCount); + for (int i = 0; i < inferenceCount; i++) { + asyncInfer("Auto scale and infer", randomFrom(idsList), TimeValue.timeValueSeconds(5), new ResponseListener() { + @Override + public void onSuccess(Response response) { + latch.countDown(); + } + + @Override + public void onFailure(Exception exception) { + latch.countDown(); + fail(exception.getMessage()); + } + }); + } + + latch.await(); + } +} diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java index 13fb2f21bbf67..f6eb7206009fa 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutoscalingIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.autoscaling.action.PutAutoscalingPolicyAction; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResults; +import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; import org.elasticsearch.xpack.core.ml.action.PutTrainedModelDefinitionPartAction; import org.elasticsearch.xpack.core.ml.action.PutTrainedModelVocabularyAction; @@ -62,14 +63,14 @@ public class AutoscalingIT extends MlNativeAutodetectIntegTestCase { @Before public void putSettings() { updateClusterSettings( - Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 100).put("logger.org.elasticsearch.xpack.ml", "DEBUG") + Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), 100).put("logger.org.elasticsearch.xpack.ml", "DEBUG") ); } @After public void removeSettings() { updateClusterSettings( - Settings.builder().putNull(MachineLearning.MAX_LAZY_ML_NODES.getKey()).putNull("logger.org.elasticsearch.xpack.ml") + Settings.builder().putNull(MachineLearningField.MAX_LAZY_ML_NODES.getKey()).putNull("logger.org.elasticsearch.xpack.ml") ); cleanUp(); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java index c785ae96c5c16..b89f1d0c77ed8 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java @@ -10,13 +10,17 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus; import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner; @@ -282,6 +286,27 @@ protected Response startDeployment( return client().performRequest(request); } + protected Response startDeployment(String modelId, String deploymentId, AdaptiveAllocationsSettings adaptiveAllocationsSettings) + throws IOException { + String endPoint = "/_ml/trained_models/" + + modelId + + "/deployment/_start" + + "?deployment_id=" + + deploymentId + + "&threads_per_allocation=1" + + "&wait_for=started"; + + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + builder.field("adaptive_allocations", adaptiveAllocationsSettings); + builder.endObject(); + var body = Strings.toString(builder); + + Request request = new Request("POST", endPoint); + request.setJsonEntity(body); + return client().performRequest(request); + } + protected void stopDeployment(String modelId) throws IOException { stopDeployment(modelId, false, false); } @@ -325,6 +350,14 @@ protected Response infer(String input, String modelId, TimeValue timeout) throws return client().performRequest(request); } + protected void asyncInfer(String input, String modelId, TimeValue timeout, ResponseListener responseListener) throws IOException { + Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=" + timeout.toString()); + request.setJsonEntity(Strings.format(""" + { "docs": [{"input":"%s"}] } + """, input)); + client().performRequestAsync(request, responseListener); + } + protected Response infer(String input, String modelId) throws IOException { Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=30s"); request.setJsonEntity(Strings.format(""" diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index f6a58002bbac5..083a444fbf14c 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; @@ -84,9 +85,9 @@ public void testLazyNodeValidation() throws Exception { logger.info("Started [{}] nodes", numNodes); ensureStableCluster(numNodes); ensureTemplatesArePresent(); - logger.info("[{}] is [{}]", MachineLearning.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes); + logger.info("[{}] is [{}]", MachineLearningField.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes); // Set our lazy node number - updateClusterSettings(Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes)); + updateClusterSettings(Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), maxNumberOfLazyNodes)); // create and open first job, which succeeds: Job.Builder job = createJob("lazy-node-validation-job-1", ByteSizeValue.ofMb(2)); PutJobAction.Request putJobRequest = new PutJobAction.Request(job); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index f8a590a23a2c1..6d21654f9e161 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -649,14 +649,6 @@ public void loadExtensions(ExtensionLoader loader) { Property.NodeScope ); - public static final Setting MAX_LAZY_ML_NODES = Setting.intSetting( - "xpack.ml.max_lazy_ml_nodes", - 0, - 0, - Property.OperatorDynamic, - Property.NodeScope - ); - // Before 8.0.0 this needs to match the max allowed value for xpack.ml.max_open_jobs, // as the current node could be running in a cluster where some nodes are still using // that setting. From 8.0.0 onwards we have the flexibility to increase it... @@ -810,7 +802,7 @@ public List> getSettings() { PROCESS_CONNECT_TIMEOUT, CONCURRENT_JOB_ALLOCATIONS, MachineLearningField.MAX_MODEL_MEMORY_LIMIT, - MAX_LAZY_ML_NODES, + MachineLearningField.MAX_LAZY_ML_NODES, MAX_MACHINE_MEMORY_PERCENT, AutodetectBuilder.MAX_ANOMALY_RECORDS_SETTING_DYNAMIC, MAX_OPEN_JOBS_PER_NODE, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java index 5603e9c4dca8d..a81a08519f157 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java @@ -11,9 +11,11 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; +import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -27,7 +29,9 @@ public TransportExternalInferModelAction( ClusterService clusterService, XPackLicenseState licenseState, TrainedModelProvider trainedModelProvider, - AdaptiveAllocationsScalerService adaptiveAllocationsScalerService + AdaptiveAllocationsScalerService adaptiveAllocationsScalerService, + TrainedModelAssignmentService assignmentService, + ThreadPool threadPool ) { super( InferModelAction.EXTERNAL_NAME, @@ -38,7 +42,9 @@ public TransportExternalInferModelAction( clusterService, licenseState, trainedModelProvider, - adaptiveAllocationsScalerService + adaptiveAllocationsScalerService, + assignmentService, + threadPool ); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java index b69f8c7d62eb2..ba4483493da1d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; @@ -42,7 +43,10 @@ import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.inference.InferenceWaitForAllocation; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; +import org.elasticsearch.xpack.ml.inference.adaptiveallocations.ScaleFromZeroFeatureFlag; +import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService; import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -68,6 +72,8 @@ public class TransportInternalInferModelAction extends HandledTransportAction { + var nodes = assignment.selectRandomNodesWeighedOnAllocations(request.request().numberOfDocuments(), RoutingState.STARTED); + + if (nodes.isEmpty()) { + request.listener() + .onFailure( + new IllegalStateException( + "[" + request.deploymentId() + "] error waiting for started allocations. The assignment has 0 started nodes" + ) + ); + } + + inferOnAssignmentNodes( + assignment.getDeploymentId(), + nodes, + request.request(), + request.responseBuilder(), + request.parentTaskId(), + request.listener() + ); + }); + } + + private void inferOnAssignmentNodes( + String deploymentId, + List> nodes, + Request request, + Response.Builder responseBuilder, + TaskId parentTaskId, + ActionListener listener + ) { AtomicInteger count = new AtomicInteger(); AtomicArray> results = new AtomicArray<>(nodes.size()); AtomicReference failure = new AtomicReference<>(); @@ -282,14 +339,14 @@ private void inferAgainstAllocatedModel( InferTrainedModelDeploymentAction.Request deploymentRequest; if (request.getTextInput() == null) { deploymentRequest = InferTrainedModelDeploymentAction.Request.forDocs( - assignment.getDeploymentId(), + deploymentId, request.getUpdate(), request.getObjectsToInfer().subList(startPos, startPos + node.v2()), request.getInferenceTimeout() ); } else { deploymentRequest = InferTrainedModelDeploymentAction.Request.forTextInput( - assignment.getDeploymentId(), + deploymentId, request.getUpdate(), request.getTextInput().subList(startPos, startPos + node.v2()), request.getInferenceTimeout() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java index bc017915e00aa..1edc02ff44a11 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.MlInfoAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -162,7 +163,7 @@ private Map limits() { clusterSettings.get(MachineLearning.ALLOCATED_PROCESSORS_SCALE) ); if (totalMlProcessors.count() > 0) { - int potentialExtraProcessors = Math.max(0, clusterSettings.get(MachineLearning.MAX_LAZY_ML_NODES) - mlNodes.size()) + int potentialExtraProcessors = Math.max(0, clusterSettings.get(MachineLearningField.MAX_LAZY_ML_NODES) - mlNodes.size()) * singleNodeProcessors.roundUp(); limits.put("total_ml_processors", totalMlProcessors.roundUp() + potentialExtraProcessors); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java index e130b13f4ec30..0bda2de2ce9ae 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java @@ -234,9 +234,9 @@ protected void masterOperation( if (getModelResponse.getResources().results().size() > 1) { listener.onFailure( ExceptionsHelper.badRequestException( - "cannot deploy more than one models at the same time; [{}] matches [{}] models]", + "cannot deploy more than one model at the same time; [{}] matches models [{}]", request.getModelId(), - getModelResponse.getResources().results().size() + getModelResponse.getResources().results().stream().map(TrainedModelConfig::getModelId).toList() ) ); return; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/InferenceWaitForAllocation.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/InferenceWaitForAllocation.java new file mode 100644 index 0000000000000..1142257a087c6 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/InferenceWaitForAllocation.java @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.inference; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xpack.core.ml.action.InferModelAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; +import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState; +import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment; +import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.Predicate; + +import static org.elasticsearch.core.Strings.format; + +/** + * Class for storing inference requests for ml trained models while + * scaling is in progress. Once the trained model has at least 1 + * allocation the stored requests are forwarded to a consumer for + * processing.Requests will timeout while waiting for scale. + */ +public class InferenceWaitForAllocation { + + public static final int MAX_PENDING_REQUEST_COUNT = 100; + + /** + * Track details of the pending request + */ + public record WaitingRequest( + InferModelAction.Request request, + InferModelAction.Response.Builder responseBuilder, + TaskId parentTaskId, + ActionListener listener + ) { + public String deploymentId() { + return request.getId(); + } + } + + private static final Logger logger = LogManager.getLogger(InferenceWaitForAllocation.class); + + private final TrainedModelAssignmentService assignmentService; + private final BiConsumer queuedConsumer; + private AtomicInteger pendingRequestCount = new AtomicInteger(); + + /** + * Create with consumer of the successful requests + * @param assignmentService Trained model assignment service + * @param onInferenceScaledConsumer The consumer of the waiting request called once an + * allocation is available. + */ + public InferenceWaitForAllocation( + TrainedModelAssignmentService assignmentService, + BiConsumer onInferenceScaledConsumer + ) { + this.assignmentService = assignmentService; + this.queuedConsumer = onInferenceScaledConsumer; + } + + /** + * Wait for at least 1 allocation to be started then process the + * inference request. + * If the pending request count is greater than {@link #MAX_PENDING_REQUEST_COUNT} + * the request listener is failed with a too many requests exception + * The timeout is the inference request timeout. + * @param request The inference request details + */ + public synchronized void waitForAssignment(WaitingRequest request) { + if (pendingRequestCount.incrementAndGet() >= MAX_PENDING_REQUEST_COUNT) { + pendingRequestCount.decrementAndGet(); + request.listener.onFailure( + new ElasticsearchStatusException( + "Rejected inference request waiting for an allocation of deployment [{}]. Too many pending requests", + RestStatus.TOO_MANY_REQUESTS, + request.request.getId() + ) + ); + return; + } + + var predicate = new DeploymentHasAtLeastOneAllocation(request.deploymentId()); + + assignmentService.waitForAssignmentCondition( + request.deploymentId(), + predicate, + request.request().getInferenceTimeout(), + new WaitingListener(request, predicate) + ); + } + + private static class DeploymentHasAtLeastOneAllocation implements Predicate { + + private final String deploymentId; + private AtomicReference exception = new AtomicReference<>(); + + DeploymentHasAtLeastOneAllocation(String deploymentId) { + this.deploymentId = ExceptionsHelper.requireNonNull(deploymentId, "deployment_id"); + } + + @Override + public boolean test(ClusterState clusterState) { + TrainedModelAssignment trainedModelAssignment = TrainedModelAssignmentMetadata.assignmentForDeploymentId( + clusterState, + deploymentId + ).orElse(null); + if (trainedModelAssignment == null) { + logger.info(() -> format("[%s] assignment was null while waiting to scale up", deploymentId)); + exception.set( + new ElasticsearchStatusException( + "[{}] Error waiting for a model allocation, model assignment has been removed", + RestStatus.CONFLICT, + deploymentId + ) + ); + return true; // don't try again + } + + Map nodeFailuresAndReasons = new HashMap<>(); + for (var nodeIdAndRouting : trainedModelAssignment.getNodeRoutingTable().entrySet()) { + if (RoutingState.FAILED.equals(nodeIdAndRouting.getValue().getState())) { + nodeFailuresAndReasons.put(nodeIdAndRouting.getKey(), nodeIdAndRouting.getValue().getReason()); + } + } + if (nodeFailuresAndReasons.isEmpty() == false) { + if (nodeFailuresAndReasons.size() == trainedModelAssignment.getNodeRoutingTable().size()) { + exception.set( + new ElasticsearchStatusException( + "[{}] Error waiting for a model allocation, all nodes have failed with errors [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + trainedModelAssignment.getDeploymentId(), + nodeFailuresAndReasons + ) + ); + return true; // don't try again + } else { + logger.warn("Deployment [{}] has failed routes [{}]", trainedModelAssignment.getDeploymentId(), nodeFailuresAndReasons); + } + } + + var routable = trainedModelAssignment.getNodeRoutingTable().values().stream().filter(RoutingInfo::isRoutable).findFirst(); + return routable.isPresent(); + } + } + + private class WaitingListener implements TrainedModelAssignmentService.WaitForAssignmentListener { + + private final WaitingRequest request; + private final DeploymentHasAtLeastOneAllocation predicate; + + private WaitingListener(WaitingRequest request, DeploymentHasAtLeastOneAllocation predicate) { + this.request = request; + this.predicate = predicate; + } + + @Override + public void onResponse(TrainedModelAssignment assignment) { + // assignment is started, do inference + pendingRequestCount.decrementAndGet(); + + if (predicate.exception.get() != null) { + onFailure(predicate.exception.get()); + return; + } + + queuedConsumer.accept(request, assignment); + } + + @Override + public void onFailure(Exception e) { + pendingRequestCount.decrementAndGet(); + request.listener().onFailure(e); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java index 58259b87c6b00..bbd63e0d3bfe9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.TimeValue; /** * Processes measured requests counts and inference times and decides whether @@ -22,12 +21,6 @@ public class AdaptiveAllocationsScaler { static final double SCALE_UP_THRESHOLD = 0.9; private static final double SCALE_DOWN_THRESHOLD = 0.85; - /** - * The time interval without any requests that has to pass, before scaling down - * to zero allocations (in case min_allocations = 0). - */ - private static final long SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS = TimeValue.timeValueMinutes(15).getSeconds(); - /** * If the max_number_of_allocations is not set, use this value for now to prevent scaling up * to high numbers due to possible bugs or unexpected behaviour in the scaler. @@ -51,8 +44,9 @@ public class AdaptiveAllocationsScaler { private Double lastMeasuredRequestRate; private Double lastMeasuredInferenceTime; private Long lastMeasuredQueueSize; + private long scaleToZeroAfterNoRequestsSeconds; - AdaptiveAllocationsScaler(String deploymentId, int numberOfAllocations) { + AdaptiveAllocationsScaler(String deploymentId, int numberOfAllocations, long scaleToZeroAfterNoRequestsSeconds) { this.deploymentId = deploymentId; // A smoothing factor of 100 roughly means the last 100 measurements have an effect // on the estimated values. The sampling time is 10 seconds, so approximately the @@ -73,6 +67,7 @@ public class AdaptiveAllocationsScaler { lastMeasuredRequestRate = null; lastMeasuredInferenceTime = null; lastMeasuredQueueSize = null; + this.scaleToZeroAfterNoRequestsSeconds = scaleToZeroAfterNoRequestsSeconds; } void setMinMaxNumberOfAllocations(Integer minNumberOfAllocations, Integer maxNumberOfAllocations) { @@ -143,6 +138,7 @@ Double getInferenceTimeEstimate() { } Integer scale() { + if (requestRateEstimator.hasValue() == false) { return null; } @@ -170,9 +166,14 @@ Integer scale() { if (maxNumberOfAllocations != null) { numberOfAllocations = Math.min(numberOfAllocations, maxNumberOfAllocations); } + if ((minNumberOfAllocations == null || minNumberOfAllocations == 0) - && timeWithoutRequestsSeconds > SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS) { - logger.debug("[{}] adaptive allocations scaler: scaling down to zero, because of no requests.", deploymentId); + && timeWithoutRequestsSeconds > scaleToZeroAfterNoRequestsSeconds) { + + if (oldNumberOfAllocations != 0) { + // avoid logging this message if there is no change + logger.debug("[{}] adaptive allocations scaler: scaling down to zero, because of no requests.", deploymentId); + } numberOfAllocations = 0; neededNumberOfAllocations = 0; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java index 193fa9e7e07f9..1c3a73a409dd1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java @@ -25,6 +25,7 @@ import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentStats; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; @@ -171,17 +173,6 @@ Collection observeDouble(Function deploymentIdsWithInFlightScaleFromZeroRequests = new ConcurrentSkipListSet<>(); + public AdaptiveAllocationsScalerService( ThreadPool threadPool, ClusterService clusterService, @@ -247,6 +248,7 @@ public AdaptiveAllocationsScalerService( scalers = new HashMap<>(); metrics = new Metrics(); busy = new AtomicBoolean(false); + scaleToZeroAfterNoRequestsSeconds = SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS; } public synchronized void start() { @@ -261,7 +263,7 @@ public synchronized void start() { public synchronized void stop() { clusterService.removeListener(this); stopScheduling(); - metrics.close(); + scalers.clear(); } @Override @@ -290,7 +292,11 @@ private synchronized void updateAutoscalers(ClusterState state) { && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) { AdaptiveAllocationsScaler adaptiveAllocationsScaler = scalers.computeIfAbsent( assignment.getDeploymentId(), - key -> new AdaptiveAllocationsScaler(assignment.getDeploymentId(), assignment.totalTargetAllocations()) + key -> new AdaptiveAllocationsScaler( + assignment.getDeploymentId(), + assignment.totalTargetAllocations(), + scaleToZeroAfterNoRequestsSeconds + ) ); adaptiveAllocationsScaler.setMinMaxNumberOfAllocations( assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations(), @@ -415,22 +421,42 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo if (newNumberOfAllocations > numberOfAllocations.get(deploymentId)) { lastScaleUpTimesMillis.put(deploymentId, now); } - updateNumberOfAllocations(deploymentId, newNumberOfAllocations); + updateNumberOfAllocations( + deploymentId, + newNumberOfAllocations, + updateAssigmentListener(deploymentId, newNumberOfAllocations) + ); } } } public boolean maybeStartAllocation(TrainedModelAssignment assignment) { if (assignment.getAdaptiveAllocationsSettings() != null - && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) { - lastScaleUpTimesMillis.put(assignment.getDeploymentId(), System.currentTimeMillis()); - updateNumberOfAllocations(assignment.getDeploymentId(), 1); + && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE + && assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations() == 0) { + + // Prevent against a flurry of scale up requests. + if (deploymentIdsWithInFlightScaleFromZeroRequests.contains(assignment.getDeploymentId()) == false) { + lastScaleUpTimesMillis.put(assignment.getDeploymentId(), System.currentTimeMillis()); + var updateListener = updateAssigmentListener(assignment.getDeploymentId(), 1); + var cleanUpListener = ActionListener.runAfter( + updateListener, + () -> deploymentIdsWithInFlightScaleFromZeroRequests.remove(assignment.getDeploymentId()) + ); + + deploymentIdsWithInFlightScaleFromZeroRequests.add(assignment.getDeploymentId()); + updateNumberOfAllocations(assignment.getDeploymentId(), 1, cleanUpListener); + } return true; } return false; } - private void updateNumberOfAllocations(String deploymentId, int numberOfAllocations) { + private void updateNumberOfAllocations( + String deploymentId, + int numberOfAllocations, + ActionListener listener + ) { UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(deploymentId); updateRequest.setNumberOfAllocations(numberOfAllocations); updateRequest.setIsInternal(true); @@ -439,35 +465,38 @@ private void updateNumberOfAllocations(String deploymentId, int numberOfAllocati ClientHelper.ML_ORIGIN, UpdateTrainedModelDeploymentAction.INSTANCE, updateRequest, - ActionListener.wrap(updateResponse -> { - logger.info("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, numberOfAllocations); - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) - .execute( - () -> inferenceAuditor.info( - deploymentId, - Strings.format( - "adaptive allocations scaler: scaled [%s] to [%s] allocations.", - deploymentId, - numberOfAllocations - ) - ) - ); - }, e -> { - logger.atLevel(Level.WARN) - .withThrowable(e) - .log("adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", deploymentId, numberOfAllocations); - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) - .execute( - () -> inferenceAuditor.warning( + listener + ); + } + + private ActionListener updateAssigmentListener( + String deploymentId, + int numberOfAllocations + ) { + return ActionListener.wrap(updateResponse -> { + logger.debug("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, numberOfAllocations); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute( + () -> inferenceAuditor.info( + deploymentId, + Strings.format("adaptive allocations scaler: scaled [%s] to [%s] allocations.", deploymentId, numberOfAllocations) + ) + ); + }, e -> { + logger.atLevel(Level.WARN) + .withThrowable(e) + .log("adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", deploymentId, numberOfAllocations); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute( + () -> inferenceAuditor.warning( + deploymentId, + Strings.format( + "adaptive allocations scaler: scaling [%s] to [%s] allocations failed.", deploymentId, - Strings.format( - "adaptive allocations scaler: scaling [%s] to [%s] allocations failed.", - deploymentId, - numberOfAllocations - ) + numberOfAllocations ) - ); - }) - ); + ) + ); + }); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleFromZeroFeatureFlag.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleFromZeroFeatureFlag.java new file mode 100644 index 0000000000000..4c446b65db9dd --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleFromZeroFeatureFlag.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.inference.adaptiveallocations; + +import org.elasticsearch.common.util.FeatureFlag; + +public class ScaleFromZeroFeatureFlag { + private ScaleFromZeroFeatureFlag() {} + + private static final FeatureFlag FEATURE_FLAG = new FeatureFlag("ml_scale_from_zero"); + + public static boolean isEnabled() { + return FEATURE_FLAG.isEnabled(); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java index 96490716c5c6c..0439ef9db30b4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java @@ -113,7 +113,7 @@ public TrainedModelAssignmentClusterService( this.maxMemoryPercentage = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings); this.useAuto = MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT.get(settings); this.maxOpenJobs = MachineLearning.MAX_OPEN_JOBS_PER_NODE.get(settings); - this.maxLazyMLNodes = MachineLearning.MAX_LAZY_ML_NODES.get(settings); + this.maxLazyMLNodes = MachineLearningField.MAX_LAZY_ML_NODES.get(settings); this.maxMLNodeSize = MachineLearning.MAX_ML_NODE_SIZE.get(settings).getBytes(); this.allocatedProcessorsScale = MachineLearning.ALLOCATED_PROCESSORS_SCALE.get(settings); this.client = client; @@ -125,7 +125,7 @@ public TrainedModelAssignmentClusterService( clusterService.getClusterSettings() .addSettingsUpdateConsumer(MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, this::setUseAuto); clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_OPEN_JOBS_PER_NODE, this::setMaxOpenJobs); - clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes); + clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearningField.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes); clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_ML_NODE_SIZE, this::setMaxMLNodeSize); clusterService.getClusterSettings() .addSettingsUpdateConsumer(MachineLearning.ALLOCATED_PROCESSORS_SCALE, this::setAllocatedProcessorsScale); @@ -845,10 +845,9 @@ private void updateDeployment( return; } } - boolean hasUpdates = (numberOfAllocations != null - && Objects.equals(numberOfAllocations, existingAssignment.getTaskParams().getNumberOfAllocations()) == false) - || Objects.equals(adaptiveAllocationsSettings, existingAssignment.getAdaptiveAllocationsSettings()) == false; + boolean hasUpdates = hasUpdates(numberOfAllocations, adaptiveAllocationsSettingsUpdates, existingAssignment); if (hasUpdates == false) { + logger.info("no updates"); listener.onResponse(existingAssignment); return; } @@ -917,6 +916,17 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) updateAssignment(clusterState, existingAssignment, numberOfAllocations, adaptiveAllocationsSettings, updatedStateListener); } + static boolean hasUpdates( + Integer proposedNumberOfAllocations, + AdaptiveAllocationsSettings proposedAdaptiveSettings, + TrainedModelAssignment existingAssignment + ) { + return (proposedNumberOfAllocations != null + && Objects.equals(proposedNumberOfAllocations, existingAssignment.getTaskParams().getNumberOfAllocations()) == false) + || (proposedAdaptiveSettings != null + && Objects.equals(proposedAdaptiveSettings, existingAssignment.getAdaptiveAllocationsSettings()) == false); + } + private AdaptiveAllocationsSettings getAdaptiveAllocationsSettings( AdaptiveAllocationsSettings original, AdaptiveAllocationsSettings updates diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java index 8c7c12de0ce51..48c6abde3010a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.rest.job; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -21,7 +20,6 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH; public class RestPostDataAction extends BaseRestHandler { @@ -39,8 +37,7 @@ public List routes() { + "in a future major version it will be compulsory to use a datafeed"; return List.of( // Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecated(msg, RestApiVersion.V_8).build(), - Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecateAndKeep(msg).build(), - Route.builder(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecated(msg, RestApiVersion.V_7).build() + Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecateAndKeep(msg).build() ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java index 32543b45259c2..7e0ff4f029bd4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutor.java @@ -24,6 +24,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor; +import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.MachineLearning; @@ -103,7 +104,7 @@ protected AbstractJobPersistentTasksExecutor( this.expressionResolver = Objects.requireNonNull(expressionResolver); this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings); this.maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings); - this.maxLazyMLNodes = MachineLearning.MAX_LAZY_ML_NODES.get(settings); + this.maxLazyMLNodes = MachineLearningField.MAX_LAZY_ML_NODES.get(settings); this.maxOpenJobs = MAX_OPEN_JOBS_PER_NODE.get(settings); this.useAutoMemoryPercentage = USE_AUTO_MACHINE_MEMORY_PERCENT.get(settings); this.maxNodeMemory = MAX_ML_NODE_SIZE.get(settings).getBytes(); @@ -111,7 +112,7 @@ protected AbstractJobPersistentTasksExecutor( .addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations); clusterService.getClusterSettings() .addSettingsUpdateConsumer(MachineLearning.MAX_MACHINE_MEMORY_PERCENT, this::setMaxMachineMemoryPercent); - clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes); + clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearningField.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_OPEN_JOBS_PER_NODE, this::setMaxOpenJobs); clusterService.getClusterSettings().addSettingsUpdateConsumer(USE_AUTO_MACHINE_MEMORY_PERCENT, this::setUseAutoMemoryPercentage); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_ML_NODE_SIZE, this::setMaxNodeSize); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java index 020f1aae29427..980d7d6b57481 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java @@ -22,10 +22,10 @@ import java.util.OptionalLong; +import static org.elasticsearch.xpack.core.ml.MachineLearningField.MAX_LAZY_ML_NODES; import static org.elasticsearch.xpack.core.ml.MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT; import static org.elasticsearch.xpack.ml.MachineLearning.MACHINE_MEMORY_NODE_ATTR; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_JVM_SIZE_NODE_ATTR; -import static org.elasticsearch.xpack.ml.MachineLearning.MAX_LAZY_ML_NODES; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_MACHINE_MEMORY_PERCENT; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_ML_NODE_SIZE; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java index 64d1414134f38..33fae40f80db6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java @@ -130,7 +130,7 @@ private static TaskExecutor createTaskExecutor() { MachineLearning.MAX_MACHINE_MEMORY_PERCENT, MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, MachineLearning.MAX_ML_NODE_SIZE, - MachineLearning.MAX_LAZY_ML_NODES, + MachineLearningField.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE ) ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java index 4aaddc91231f3..79f2a913902df 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java @@ -14,6 +14,8 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -38,6 +40,9 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -66,6 +71,8 @@ public void setUp() throws Exception { new ScalingExecutorBuilder(MachineLearning.UTILITY_THREAD_POOL_NAME, 0, 1, TimeValue.timeValueMinutes(10), false) ); clusterService = mock(ClusterService.class); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, Set.of())); client = mock(Client.class); inferenceAuditor = mock(InferenceAuditor.class); meterRegistry = mock(MeterRegistry.class); @@ -240,4 +247,99 @@ public void test() throws IOException { service.stop(); } + + public void testMaybeStartAllocation() { + AdaptiveAllocationsScalerService service = new AdaptiveAllocationsScalerService( + threadPool, + clusterService, + client, + inferenceAuditor, + meterRegistry, + true, + 1 + ); + + when(client.threadPool()).thenReturn(threadPool); + + // will not start when adaptive allocations are not enabled + assertFalse(service.maybeStartAllocation(TrainedModelAssignment.Builder.empty(taskParams(1), null).build())); + assertFalse( + service.maybeStartAllocation( + TrainedModelAssignment.Builder.empty(taskParams(1), new AdaptiveAllocationsSettings(Boolean.FALSE, 1, 2)).build() + ) + ); + // min allocations > 0 + assertFalse( + service.maybeStartAllocation( + TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 1, 2)).build() + ) + ); + assertTrue( + service.maybeStartAllocation( + TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 2)).build() + ) + ); + } + + public void testMaybeStartAllocation_BlocksMultipleRequests() throws Exception { + AdaptiveAllocationsScalerService service = new AdaptiveAllocationsScalerService( + threadPool, + clusterService, + client, + inferenceAuditor, + meterRegistry, + true, + 1 + ); + + var latch = new CountDownLatch(1); + var scalingUpRequestSent = new AtomicBoolean(); + + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + scalingUpRequestSent.set(true); + latch.await(); + listener.onResponse(mock(CreateTrainedModelAssignmentAction.Response.class)); + return Void.TYPE; + }).when(client).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), any(), any()); + + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> { + var starting = service.maybeStartAllocation( + TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 2)).build() + ); + assertTrue(starting); + }); + + // wait for the request to be sent + assertBusy(() -> assertTrue(scalingUpRequestSent.get())); + + // Due to the inflight request this will not trigger an update request + assertTrue( + service.maybeStartAllocation( + TrainedModelAssignment.Builder.empty(taskParams(0), new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 2)).build() + ) + ); + // release the inflight request + latch.countDown(); + + verify(client, times(1)).execute(eq(UpdateTrainedModelDeploymentAction.INSTANCE), any(), any()); + } + + private StartTrainedModelDeploymentAction.TaskParams taskParams(int numAllocations) { + return new StartTrainedModelDeploymentAction.TaskParams( + "foo", + "foo", + 1000L, + numAllocations, + 1, + 100, + null, + Priority.NORMAL, + 100L, + 100L + ); + } + } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java index 1887ebe8050e0..0fb8ad314343a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java @@ -7,7 +7,10 @@ package org.elasticsearch.xpack.ml.inference.adaptiveallocations; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; import java.util.Random; @@ -15,11 +18,21 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class AdaptiveAllocationsScalerTests extends ESTestCase { + private ClusterService clusterService; + + @Before + public void createMocks() throws Exception { + clusterService = mock(ClusterService.class); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + } + public void testAutoscaling_scaleUpAndDown() { - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60); // With 1 allocation the system can handle 500 requests * 0.020 sec/request. // To handle remaining requests the system should scale to 2 allocations. @@ -47,7 +60,7 @@ public void testAutoscaling_scaleUpAndDown() { } public void testAutoscaling_noOscillating() { - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60); // With 1 allocation the system can handle 880 requests * 0.010 sec/request. adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(880, 0, 0, 0.010), 10, 1); @@ -75,7 +88,7 @@ public void testAutoscaling_noOscillating() { } public void testAutoscaling_respectMinMaxAllocations() { - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60); adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(2, 5); // Even though there are no requests, scale to the minimum of 2 allocations. @@ -98,7 +111,7 @@ public void testAutoscaling_respectMinMaxAllocations() { } public void testEstimation_highVariance() { - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60); Random random = new Random(42); @@ -140,7 +153,7 @@ public void testEstimation_highVariance() { } public void testAutoscaling_maxAllocationsSafeguard() { - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60); adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1_000_000, 10_000_000, 1, 0.05), 10, 1); assertThat(adaptiveAllocationsScaler.scale(), equalTo(32)); adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(2, 77); @@ -148,7 +161,12 @@ public void testAutoscaling_maxAllocationsSafeguard() { } public void testAutoscaling_scaleDownToZeroAllocations() { - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + int scaleDownAfterInactivitySeconds = 60 * 15; // scale down to 0 after 15 minutes + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler( + "test-deployment", + 1, + scaleDownAfterInactivitySeconds + ); // 1 hour with 1 request per 1 seconds, so don't scale. for (int i = 0; i < 3600; i++) { adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1); @@ -178,7 +196,7 @@ public void testAutoscaling_scaleDownToZeroAllocations() { } public void testAutoscaling_dontScaleDownToZeroAllocationsWhenMinAllocationsIsSet() { - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1, 60); adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(1, null); // 1 hour with no requests, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java index 1dc44582492aa..5a358ef833cb5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java @@ -52,6 +52,7 @@ import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelAssignmentRoutingInfoAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState; import org.elasticsearch.xpack.core.ml.inference.assignment.Priority; import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo; @@ -127,7 +128,7 @@ public void setupObjects() throws IllegalAccessException { MachineLearning.MAX_MACHINE_MEMORY_PERCENT, MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, MachineLearning.MAX_OPEN_JOBS_PER_NODE, - MachineLearning.MAX_LAZY_ML_NODES, + MachineLearningField.MAX_LAZY_ML_NODES, MachineLearning.MAX_ML_NODE_SIZE, MachineLearning.ALLOCATED_PROCESSORS_SCALE ) @@ -2077,9 +2078,23 @@ private void assertThatStoppingAssignmentPreventsMutation( ); } + public void testHasUpdates() { + var assignment = TrainedModelAssignment.Builder.empty(newParams("foo", 10_000L, 1, 1), null).build(); + assertFalse(TrainedModelAssignmentClusterService.hasUpdates(1, null, assignment)); + assertTrue(TrainedModelAssignmentClusterService.hasUpdates(2, null, assignment)); + + var adaptiveAllocations = new AdaptiveAllocationsSettings(true, 1, 4); + assignment = TrainedModelAssignment.Builder.empty(newParams("foo", 10_000L, 1, 1), adaptiveAllocations).build(); + assertFalse(TrainedModelAssignmentClusterService.hasUpdates(null, new AdaptiveAllocationsSettings(true, 1, 4), assignment)); + assertTrue(TrainedModelAssignmentClusterService.hasUpdates(null, new AdaptiveAllocationsSettings(true, 0, 4), assignment)); + + assertFalse(TrainedModelAssignmentClusterService.hasUpdates(1, new AdaptiveAllocationsSettings(true, 1, 4), assignment)); + assertTrue(TrainedModelAssignmentClusterService.hasUpdates(1, new AdaptiveAllocationsSettings(true, 0, 4), assignment)); + } + private TrainedModelAssignmentClusterService createClusterService(int maxLazyNodes) { return new TrainedModelAssignmentClusterService( - Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), maxLazyNodes).build(), + Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), maxLazyNodes).build(), clusterService, threadPool, nodeLoadDetector, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java index eb2e21d5fda6c..64251c05af7c8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java @@ -105,7 +105,7 @@ public void setUpMocks() { ClusterApplierService.CLUSTER_SERVICE_SLOW_TASK_THREAD_DUMP_TIMEOUT_SETTING, MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, - MachineLearning.MAX_LAZY_ML_NODES, + MachineLearningField.MAX_LAZY_ML_NODES, MachineLearning.MAX_ML_NODE_SIZE, MachineLearning.MAX_OPEN_JOBS_PER_NODE, MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT @@ -155,7 +155,7 @@ public void testValidate_givenValidJob() { // An index being unavailable should take precedence over waiting for a lazy node public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { - Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 1).build(); + Settings settings = Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), 1).build(); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); Metadata.Builder metadata = Metadata.builder(); @@ -177,7 +177,7 @@ public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { } public void testGetAssignment_GivenLazyJobAndNoGlobalLazyNodes() { - Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 0).build(); + Settings settings = Settings.builder().put(MachineLearningField.MAX_LAZY_ML_NODES.getKey(), 0).build(); ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); Metadata.Builder metadata = Metadata.builder(); RoutingTable.Builder routingTable = RoutingTable.builder(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java index 7f7c22594abb8..fdb4458a6ec3f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculatorTests.java @@ -36,11 +36,11 @@ import java.util.Set; import java.util.function.BiConsumer; +import static org.elasticsearch.xpack.core.ml.MachineLearningField.MAX_LAZY_ML_NODES; import static org.elasticsearch.xpack.core.ml.MachineLearningField.MAX_MODEL_MEMORY_LIMIT; import static org.elasticsearch.xpack.core.ml.MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT; import static org.elasticsearch.xpack.ml.MachineLearning.MACHINE_MEMORY_NODE_ATTR; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_JVM_SIZE_NODE_ATTR; -import static org.elasticsearch.xpack.ml.MachineLearning.MAX_LAZY_ML_NODES; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_MACHINE_MEMORY_PERCENT; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_ML_NODE_SIZE; import static org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingDeciderServiceTests.AUTO_NODE_TIERS_NO_MONITORING; diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index 513e1a857787e..4a039886ecc4e 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -17,6 +17,9 @@ template: type: constant_keyword data_stream.namespace: type: constant_keyword + event.dataset: + type: alias + path: data_stream.dataset attributes: type: passthrough dynamic: true diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index 0957a79552ad3..be4de6dca6c76 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -145,3 +145,21 @@ Structured log body: index: $datastream-backing-index - is_true: $datastream-backing-index - match: { .$datastream-backing-index.mappings.properties.body.properties.flattened.type: "flattened" } +--- +"event.dataset alias must point to data_stream.dataset": + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}' + - is_false: errors + - is_false: errors + - do: + search: + index: logs-generic.otel-default + body: + fields: ["event.dataset"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.event\.dataset: ["generic.otel"] } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java index 3943ddd3e207a..73e47a631de96 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/QlStatusResponse.java @@ -121,9 +121,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("is_running", isRunning); builder.field("is_partial", isPartial); if (startTimeMillis != null) { // start time is available only for a running eql search - builder.timeField("start_time_in_millis", "start_time", startTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("start_time_in_millis", "start_time", startTimeMillis); } - builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("expiration_time_in_millis", "expiration_time", expirationTimeMillis); if (isRunning == false) { // completion status is available only for a completed eql search builder.field("completion_status", completionStatus.getStatus()); } diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index d791873eb3142..4405ef575b24f 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -245,6 +245,7 @@ public class Constants { "cluster:admin/xpack/query_rules/get", "cluster:admin/xpack/query_rules/list", "cluster:admin/xpack/query_rules/put", + "cluster:admin/xpack/query_rules/test", "cluster:admin/xpack/rollup/delete", "cluster:admin/xpack/rollup/put", "cluster:admin/xpack/rollup/start", @@ -502,6 +503,9 @@ public class Constants { "indices:admin/data_stream/lifecycle/get", "indices:admin/data_stream/lifecycle/put", "indices:admin/data_stream/lifecycle/explain", + "indices:admin/data_stream/options/delete", + "indices:admin/data_stream/options/get", + "indices:admin/data_stream/options/put", "indices:admin/delete", "indices:admin/flush", "indices:admin/flush[s]", diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java index 55562c8ee0138..019b1e5095627 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -44,12 +43,8 @@ public RestPutRoleMappingAction(Settings settings, XPackLicenseState licenseStat @Override public List routes() { return List.of( - Route.builder(POST, "/_security/role_mapping/{name}") - .replaces(POST, "/_xpack/security/role_mapping/{name}", RestApiVersion.V_7) - .build(), - Route.builder(PUT, "/_security/role_mapping/{name}") - .replaces(PUT, "/_xpack/security/role_mapping/{name}", RestApiVersion.V_7) - .build() + Route.builder(POST, "/_security/role_mapping/{name}").build(), + Route.builder(PUT, "/_security/role_mapping/{name}").build() ); } diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java index d12d093dd5b8d..ee7438dfca428 100644 --- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java +++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java @@ -327,11 +327,8 @@ public void testAutoExpandDuringRestart() throws Exception { ensureGreen("myindex"); putNodeShutdown(primaryNodeId, SingleNodeShutdownMetadata.Type.RESTART, null); - // registering node shutdown entry does not perform reroute, neither should it. - // we provoke it here in the test to ensure that auto-expansion has run. - updateIndexSettings(Settings.builder().put("index.routing.allocation.exclude.name", "non-existent"), "myindex"); - assertBusy(() -> assertIndexSetting("myindex", "index.number_of_replicas", "1")); + assertIndexSetting("myindex", "index.number_of_replicas", "1"); indexRandomData("myindex"); internalCluster().restartNode(primaryNode, new InternalTestCluster.RestartCallback() { @@ -361,9 +358,6 @@ public void testAutoExpandDuringReplace() throws Exception { var replacementNodeName = "node_t2"; putNodeShutdown(nodeIdToReplace, SingleNodeShutdownMetadata.Type.REPLACE, replacementNodeName); - // registering node shutdown entry does not perform reroute, neither should it. - // we provoke it here in the test to ensure that auto-expansion has run. - updateIndexSettings(Settings.builder().put("index.routing.allocation.exclude.name", "non-existent"), "index"); ensureGreen("index"); assertIndexSetting("index", "index.number_of_replicas", "1"); @@ -381,6 +375,32 @@ public void testAutoExpandDuringReplace() throws Exception { assertIndexSetting("index", "index.number_of_replicas", "1"); } + public void testAutoExpandDuringShutdown() throws Exception { + + var node1 = internalCluster().startNode(); + var node2 = internalCluster().startNode(); + + createIndex("index", indexSettings(1, 0).put("index.auto_expand_replicas", randomFrom("0-all", "0-1")).build()); + indexRandomData("index"); + + ensureGreen("index"); + assertIndexSetting("index", "index.number_of_replicas", "1"); + + var nodeNameToShutdown = randomFrom(node1, node2); + var nodeIdToShutdown = getNodeId(nodeNameToShutdown); + + putNodeShutdown(nodeIdToShutdown, SingleNodeShutdownMetadata.Type.REMOVE, null); + + ensureGreen("index"); + assertIndexSetting("index", "index.number_of_replicas", "0"); + + assertBusy(() -> assertNodeShutdownStatus(nodeIdToShutdown, COMPLETE)); + internalCluster().stopNode(nodeIdToShutdown); + + ensureGreen("index"); + assertIndexSetting("index", "index.number_of_replicas", "0"); + } + public void testNodeShutdownWithUnassignedShards() throws Exception { final String nodeA = internalCluster().startNode(); final String nodeAId = getNodeId(nodeA); diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java index 810bd8f6e9ceb..95fd97cd5931f 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java @@ -122,7 +122,7 @@ public Iterator toXContentChunked(ToXContent.Params params metadata.getAllocationDelay().getStringRep() ); } - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( SingleNodeShutdownMetadata.STARTED_AT_MILLIS_FIELD.getPreferredName(), SingleNodeShutdownMetadata.STARTED_AT_READABLE_FIELD, metadata.getStartedAtMillis() @@ -138,7 +138,7 @@ public Iterator toXContentChunked(ToXContent.Params params builder.field(TARGET_NODE_NAME_FIELD.getPreferredName(), metadata.getTargetNodeName()); } if (metadata.getGracePeriod() != null) { - builder.timeField( + builder.timestampField( SingleNodeShutdownMetadata.GRACE_PERIOD_FIELD.getPreferredName(), metadata.getGracePeriod().getStringRep() ); diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java index 60fdba2051041..8426ad491e353 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryItem.java @@ -220,7 +220,7 @@ public final void writeTo(StreamOutput out) throws IOException { public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - builder.timeField(TIMESTAMP.getPreferredName(), "timestamp_string", timestamp); + builder.timestampFieldsFromUnixEpochMillis(TIMESTAMP.getPreferredName(), "timestamp_string", timestamp); builder.field(POLICY_ID.getPreferredName(), policyId); builder.field(REPOSITORY.getPreferredName(), repository); builder.field(SNAPSHOT_NAME.getPreferredName(), snapshotName); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java index 90130811c1218..143d2671b9eab 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseChunk.java @@ -158,7 +158,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.timeField("timestamp_in_millis", "timestamp", timestampMillis); + builder.timestampFieldsFromUnixEpochMillis("timestamp_in_millis", "timestamp", timestampMillis); if (anomaly() != null) { builder.field("anomaly", anomaly()); diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index d8534b963c2d7..20fb342aa4b38 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -1350,8 +1350,8 @@ private void setupDataForDateTimeTests(long randomLongDate, Long randomLongDateN indexSimpleDocumentWithBooleanValues("1", true, randomLongDate, randomLongDateNanos); index("test", "2", builder -> { - builder.timeField("test_date", null); - builder.timeField("test_date_nanos", null); + builder.timestampField("test_date", null); + builder.timestampField("test_date_nanos", null); }); } diff --git a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index 556a417fb5e79..988ee93bda6b4 100644 --- a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -43,6 +43,7 @@ public class XPackRestIT extends AbstractXPackRestTest { .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .user("x_pack_rest_user", "x-pack-test-password") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) .configFile("service_tokens", Resource.fromClasspath("service_tokens")) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml index 33c9cc7558672..83234901ae8f2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml @@ -155,3 +155,19 @@ setup: esql.query: body: query: 'FROM test [metadata _source] | STATS COUNT_DISTINCT(_source)' + +--- +"sort on _source not allowed": + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [sorting_on_source_and_counters_forbidden] + reason: "Sorting on _source shouldn't have been possible" + - do: + catch: /cannot sort on _source/ + esql.query: + body: + query: 'FROM test metadata _source | sort _source' diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 642407ac6d45b..ebf464ba667db 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -178,11 +178,19 @@ cast counter then filter: --- sort on counter without cast: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [sorting_on_source_and_counters_forbidden] + reason: "Sorting on counters shouldn't have been possible" - do: - catch: bad_request + catch: /cannot sort on counter_long/ esql.query: body: - query: 'from test | KEEP k8s.pod.network.tx | sort @k8s.pod.network.tx | limit 1' + query: 'from test | KEEP k8s.pod.network.tx | sort k8s.pod.network.tx | limit 1' --- cast then sort on counter: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml index 11be68cc764e2..cdc69001d33ef 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml @@ -39,23 +39,4 @@ } - match: { error.reason: "Unknown task_type [bad]" } ---- -"Test get all": - - do: - inference.get: - inference_id: "*" - - length: { endpoints: 1} - - match: { endpoints.0.inference_id: ".elser-2" } - - - do: - inference.get: - inference_id: _all - - length: { endpoints: 1} - - match: { endpoints.0.inference_id: ".elser-2" } - - - do: - inference.get: - inference_id: "" - - length: { endpoints: 1} - - match: { endpoints.0.inference_id: ".elser-2" } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java index f1a6d7b07d8e7..79470f967ab3c 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Email.java @@ -141,7 +141,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (priority != null) { builder.field(Field.PRIORITY.getPreferredName(), priority.value()); } - builder.timeField(Field.SENT_DATE.getPreferredName(), sentDate); + builder.timestampField(Field.SENT_DATE.getPreferredName(), sentDate); if (to != null) { builder.field(Field.TO.getPreferredName(), to, params); } diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java index fe62d4e2d2639..0b40828b8e86c 100644 --- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java @@ -48,6 +48,7 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest .setting("xpack.security.autoconfiguration.enabled", "false") .user(USER, PASS) .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public CoreWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle index 43d6d9463e0d1..986baf867b501 100644 --- a/x-pack/qa/runtime-fields/build.gradle +++ b/x-pack/qa/runtime-fields/build.gradle @@ -44,6 +44,7 @@ subprojects { setting 'xpack.security.enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") + requiresFeature 'es.sub_objects_auto_feature_flag_enabled', Version.fromString("8.16.0") } tasks.named("yamlRestTest").configure {