diff --git a/docs/changelog/110928.yaml b/docs/changelog/110928.yaml new file mode 100644 index 0000000000000..dcb2df6e6cca9 --- /dev/null +++ b/docs/changelog/110928.yaml @@ -0,0 +1,5 @@ +pr: 110928 +summary: Dense vector field types updatable for int4 +area: Vector Search +type: enhancement +issues: [] diff --git a/docs/changelog/111015.yaml b/docs/changelog/111015.yaml new file mode 100644 index 0000000000000..3cc363c8bbf6b --- /dev/null +++ b/docs/changelog/111015.yaml @@ -0,0 +1,15 @@ +pr: 111015 +summary: Always allow rebalancing by default +area: Allocation +type: enhancement +issues: [] +highlight: + title: Always allow rebalancing by default + body: |- + In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to + `indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was + appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has + better support for rebalancing a cluster that is not in `green` health, and expects to be able to rebalance some + shards away from over-full nodes to avoid allocating shards to undesirable locations in the first place. From + version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy allocator is explicitly enabled. + notable: true diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc index 1e425c77d1264..dc53837125ee9 100644 --- a/docs/reference/modules/cluster/shards_allocation.asciidoc +++ b/docs/reference/modules/cluster/shards_allocation.asciidoc @@ -98,9 +98,9 @@ the cluster: Specify when shard rebalancing is allowed: -* `always` - Always allow rebalancing. +* `always` - (default) Always allow rebalancing. * `indices_primaries_active` - Only when all primaries in the cluster are allocated. -* `indices_all_active` - (default) Only when all shards (primaries and replicas) in the cluster are allocated. +* `indices_all_active` - Only when all shards (primaries and replicas) in the cluster are allocated. -- `cluster.routing.rebalance.enable`:: diff --git a/docs/reference/release-notes/8.12.0.asciidoc b/docs/reference/release-notes/8.12.0.asciidoc index 4c0fc50584b9f..bfa99401f41a2 100644 --- a/docs/reference/release-notes/8.12.0.asciidoc +++ b/docs/reference/release-notes/8.12.0.asciidoc @@ -14,6 +14,13 @@ there are deleted documents in the segments, quantiles may fail to build and pre This issue is fixed in 8.12.1. +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + [[breaking-8.12.0]] [float] === Breaking changes diff --git a/docs/reference/release-notes/8.12.1.asciidoc b/docs/reference/release-notes/8.12.1.asciidoc index 9aa9a11b3bf02..8ebe5cbac3852 100644 --- a/docs/reference/release-notes/8.12.1.asciidoc +++ b/docs/reference/release-notes/8.12.1.asciidoc @@ -3,6 +3,16 @@ Also see <>. +[[known-issues-8.12.1]] +[float] +=== Known issues +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + [[bug-8.12.1]] [float] === Bug fixes diff --git a/docs/reference/release-notes/8.12.2.asciidoc b/docs/reference/release-notes/8.12.2.asciidoc index 2be8449b6c1df..44202ee8226eb 100644 --- a/docs/reference/release-notes/8.12.2.asciidoc +++ b/docs/reference/release-notes/8.12.2.asciidoc @@ -3,6 +3,16 @@ Also see <>. +[[known-issues-8.12.2]] +[float] +=== Known issues +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + [[bug-8.12.2]] [float] === Bug fixes diff --git a/docs/reference/release-notes/8.13.0.asciidoc b/docs/reference/release-notes/8.13.0.asciidoc index 197a417e0eff4..75e2341f33766 100644 --- a/docs/reference/release-notes/8.13.0.asciidoc +++ b/docs/reference/release-notes/8.13.0.asciidoc @@ -21,12 +21,12 @@ This affects clusters running version 8.10 or later, with an active downsampling https://www.elastic.co/guide/en/elasticsearch/reference/current/downsampling-ilm.html[configuration] or a configuration that was activated at some point since upgrading to version 8.10 or later. -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.13.1.asciidoc b/docs/reference/release-notes/8.13.1.asciidoc index f176c124e5e3b..c654af3dd5cc0 100644 --- a/docs/reference/release-notes/8.13.1.asciidoc +++ b/docs/reference/release-notes/8.13.1.asciidoc @@ -6,12 +6,12 @@ Also see <>. [[known-issues-8.13.1]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.13.2.asciidoc b/docs/reference/release-notes/8.13.2.asciidoc index c4340a200e0c5..f4540343ca9ea 100644 --- a/docs/reference/release-notes/8.13.2.asciidoc +++ b/docs/reference/release-notes/8.13.2.asciidoc @@ -6,12 +6,12 @@ Also see <>. [[known-issues-8.13.2]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.13.3.asciidoc b/docs/reference/release-notes/8.13.3.asciidoc index 759b879e16685..f1bb4211f4676 100644 --- a/docs/reference/release-notes/8.13.3.asciidoc +++ b/docs/reference/release-notes/8.13.3.asciidoc @@ -13,12 +13,12 @@ SQL:: [[known-issues-8.13.3]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.13.4.asciidoc b/docs/reference/release-notes/8.13.4.asciidoc index d8d0b632c734a..446aae048945b 100644 --- a/docs/reference/release-notes/8.13.4.asciidoc +++ b/docs/reference/release-notes/8.13.4.asciidoc @@ -6,12 +6,12 @@ Also see <>. [[known-issues-8.13.4]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.14.0.asciidoc b/docs/reference/release-notes/8.14.0.asciidoc index 87b931fd05906..c2fee6ecaa07a 100644 --- a/docs/reference/release-notes/8.14.0.asciidoc +++ b/docs/reference/release-notes/8.14.0.asciidoc @@ -15,12 +15,12 @@ Security:: [[known-issues-8.14.0]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.14.1.asciidoc b/docs/reference/release-notes/8.14.1.asciidoc index b35c1e651c767..de3ecd210b488 100644 --- a/docs/reference/release-notes/8.14.1.asciidoc +++ b/docs/reference/release-notes/8.14.1.asciidoc @@ -7,12 +7,12 @@ Also see <>. [[known-issues-8.14.1]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.14.2.asciidoc b/docs/reference/release-notes/8.14.2.asciidoc index 9c21cf6de466c..f3f0651508dca 100644 --- a/docs/reference/release-notes/8.14.2.asciidoc +++ b/docs/reference/release-notes/8.14.2.asciidoc @@ -6,12 +6,12 @@ Also see <>. [[known-issues-8.14.2]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` diff --git a/docs/reference/release-notes/8.14.3.asciidoc b/docs/reference/release-notes/8.14.3.asciidoc index 0d7d2d9d599c1..17c53faa4a37f 100644 --- a/docs/reference/release-notes/8.14.3.asciidoc +++ b/docs/reference/release-notes/8.14.3.asciidoc @@ -3,6 +3,16 @@ Also see <>. +[[known-issues-8.14.3]] +[float] +=== Known issues +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + [[bug-8.14.3]] [float] === Bug fixes diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 4976e5e15adbe..429fdba2a0562 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -114,42 +114,44 @@ using default timestamp field mapping: message: type: text ---- -missing hostname field: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" - - - do: - catch: bad_request - indices.create: - index: test-hostname-missing - body: - settings: - index: - mode: logs - number_of_replicas: 0 - number_of_shards: 2 - mappings: - properties: - "@timestamp": - type: date - agent_id: - type: keyword - process_id: - type: integer - http_method: - type: keyword - message: - type: text - - - match: { error.root_cause.0.type: "illegal_argument_exception" } - - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "unknown index sort field:[host.name]" } +#--- +#missing hostname field: +# - requires: +# test_runner_features: [ capabilities ] +# capabilities: +# - method: PUT +# path: /{index} +# capabilities: [ logs_index_mode ] +# reason: "Support for 'logs' index mode capability required" +# +# - do: +# indices.create: +# index: test-hostname-missing +# body: +# settings: +# index: +# mode: logs +# number_of_replicas: 0 +# number_of_shards: 2 +# mappings: +# properties: +# "@timestamp": +# type: date +# agent_id: +# type: keyword +# process_id: +# type: integer +# http_method: +# type: keyword +# message: +# type: text +# +# - do: +# indices.get_settings: +# index: test-hostname-missing +# +# - is_true: test-hostname-missing +# - match: { test-hostname-missing.settings.index.mode: "logs" } --- missing sort field: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml index 3502a5e643087..855daeaa7f163 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml @@ -2,9 +2,8 @@ setup: - requires: cluster_features: "gte_v8.15.0" reason: 'updatable dense vector field types was added in 8.15' - - skip: - reason: "contains is a newly added assertion" - features: contains + - requires: + test_runner_features: [ contains ] --- "Test create and update dense vector mapping with per-doc indexing and flush": - do: @@ -1016,6 +1015,45 @@ setup: index_options: type: int8_flat +--- +"Disallowed dense vector update path hnsw --> int4_flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: hnsw + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: hnsw } + + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + --- "Disallowed dense vector update path int8_hnsw --> flat": - do: @@ -1088,6 +1126,67 @@ setup: index_options: type: int8_flat +--- +"Disallowed dense vector update path int4_hnsw --> int8_flat, int4_flat, flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_flat + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: flat + --- "Disallowed dense vector update path int8_flat --> flat": - do: @@ -1124,6 +1223,56 @@ setup: index_options: type: flat +--- +"Disallowed dense vector update path int4_flat --> flat, int8_flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: flat + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_flat + --- "Allowed dense vector updates on same type but different other index_options, hnsw": - do: @@ -1320,6 +1469,103 @@ setup: ef_construction: 200 confidence_interval: 0.3 +--- +"Allowed dense vector updates on same type but different other index_options, int4_hnsw": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - requires: + test_runner_features: [ contains ] + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + m: 32 + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + - match: { test_index.mappings.properties.embedding.index_options.m: 32 } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + m: 32 + ef_construction: 200 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + - match: { test_index.mappings.properties.embedding.index_options.m: 32 } + - match: { test_index.mappings.properties.embedding.index_options.ef_construction: 200 } + + - do: + catch: /illegal_argument_exception/ # fails because m = 10 is less than the current value of 32 + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_hnsw + ef_construction: 200 + m: 10 + + - do: + catch: /illegal_argument_exception/ # fails because m = 16 by default, which is less than the current value of 32 + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_hnsw + ef_construction: 200 + --- "Allowed dense vector updates on same type but different other index_options, int8_flat": - do: @@ -1363,3 +1609,492 @@ setup: - match: { test_index.mappings.properties.embedding.type: dense_vector } - match: { test_index.mappings.properties.embedding.index_options.type: int8_flat } - match: { test_index.mappings.properties.embedding.index_options.confidence_interval: 0.3 } + +--- +"Allowed dense vector updates on same type but different other index_options, int4_flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - requires: + test_runner_features: [ contains ] + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + confidence_interval: 0.3 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + - match: { test_index.mappings.properties.embedding.index_options.confidence_interval: 0.3 } + +--- +"Test create and update dense vector mapping to int4 with per-doc indexing and flush": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - requires: + test_runner_features: [ contains ] + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: flat } + + - do: + index: + index: test_index + id: "1" + body: + embedding: [ 1, 1, 1, 1 ] + - do: + index: + index: test_index + id: "2" + body: + embedding: [ 1, 1, 1, 2 ] + - do: + index: + index: test_index + id: "3" + body: + embedding: [ 1, 1, 1, 3 ] + - do: + index: + index: test_index + id: "4" + body: + embedding: [ 1, 1, 1, 4 ] + - do: + index: + index: test_index + id: "5" + body: + embedding: [ 1, 1, 1, 5 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "6" + body: + embedding: [ 1, 1, 1, 6 ] + - do: + index: + index: test_index + id: "7" + body: + embedding: [ 1, 1, 1, 7 ] + - do: + index: + index: test_index + id: "8" + body: + embedding: [ 1, 1, 1, 8 ] + - do: + index: + index: test_index + id: "9" + body: + embedding: [ 1, 1, 1, 9 ] + - do: + index: + index: test_index + id: "10" + body: + embedding: [ 1, 1, 1, 10 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 3 + query: + knn: + field: embedding + query_vector: [1, 1, 1, 1] + num_candidates: 10 + + - match: { hits.total.value: 10 } + - length: {hits.hits: 3} + - contains: { hits.hits: { _id: "1" } } + - contains: { hits.hits: { _id: "2" } } + - contains: { hits.hits: { _id: "3" } } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + + - do: + index: + index: test_index + id: "11" + body: + embedding: [ 2, 1, 1, 1 ] + - do: + index: + index: test_index + id: "12" + body: + embedding: [ 3, 1, 1, 2 ] + - do: + index: + index: test_index + id: "13" + body: + embedding: [ 4, 1, 1, 3 ] + - do: + index: + index: test_index + id: "14" + body: + embedding: [ 5, 1, 1, 4 ] + - do: + index: + index: test_index + id: "15" + body: + embedding: [ 6, 1, 1, 5 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "16" + body: + embedding: [ 7, 1, 1, 6 ] + - do: + index: + index: test_index + id: "17" + body: + embedding: [ 8, 1, 1, 7 ] + - do: + index: + index: test_index + id: "18" + body: + embedding: [ 9, 1, 1, 8 ] + - do: + index: + index: test_index + id: "19" + body: + embedding: [ 10, 1, 1, 9 ] + - do: + index: + index: test_index + id: "20" + body: + embedding: [ 1, 11, 1, 10 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 3 + query: + knn: + field: embedding + query_vector: [ 1, 1, 1, 1 ] + num_candidates: 20 + + - match: { hits.total.value: 20 } + - length: { hits.hits: 3 } + - contains: { hits.hits: { _id: "1" } } + - contains: { hits.hits: { _id: "11" } } + - contains: { hits.hits: { _id: "2" } } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_hnsw + m: 3 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int8_hnsw } + + - do: + index: + index: test_index + id: "21" + body: + embedding: [ 1, 1, 2, 1 ] + - do: + index: + index: test_index + id: "22" + body: + embedding: [ 1, 1, 3, 1 ] + - do: + index: + index: test_index + id: "23" + body: + embedding: [ 1, 1, 4, 1 ] + - do: + index: + index: test_index + id: "24" + body: + embedding: [ 1, 1, 5, 1 ] + - do: + index: + index: test_index + id: "25" + body: + embedding: [ 1, 1, 6, 1 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "26" + body: + embedding: [ 1, 1, 7, 1 ] + - do: + index: + index: test_index + id: "27" + body: + embedding: [ 1, 1, 8, 1 ] + - do: + index: + index: test_index + id: "28" + body: + embedding: [ 1, 1, 9, 1 ] + - do: + index: + index: test_index + id: "29" + body: + embedding: [ 1, 1, 10, 1 ] + - do: + index: + index: test_index + id: "30" + body: + embedding: [ 1, 1, 11, 1 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 4 + query: + knn: + field: embedding + query_vector: [ 1, 1, 1, 1 ] + num_candidates: 30 + + - match: { hits.total.value: 30 } + - length: { hits.hits: 4 } + - contains: {hits.hits: {_id: "1"}} + - contains: {hits.hits: {_id: "11"}} + - contains: {hits.hits: {_id: "2"}} + - contains: {hits.hits: {_id: "21"}} + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + ef_construction: 200 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + + - do: + index: + index: test_index + id: "31" + body: + embedding: [ 1, 1, 1, 2 ] + - do: + index: + index: test_index + id: "32" + body: + embedding: [ 1, 1, 1, 3 ] + - do: + index: + index: test_index + id: "33" + body: + embedding: [ 1, 1, 1, 4 ] + - do: + index: + index: test_index + id: "34" + body: + embedding: [ 1, 1, 1, 5 ] + - do: + index: + index: test_index + id: "35" + body: + embedding: [ 1, 1, 1, 6 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "36" + body: + embedding: [ 1, 1, 1, 7 ] + - do: + index: + index: test_index + id: "37" + body: + embedding: [ 1, 1, 1, 8 ] + - do: + index: + index: test_index + id: "38" + body: + embedding: [ 1, 1, 1, 9 ] + - do: + index: + index: test_index + id: "39" + body: + embedding: [ 1, 1, 1, 10 ] + - do: + index: + index: test_index + id: "40" + body: + embedding: [ 1, 1, 1, 11 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 5 + query: + knn: + field: embedding + query_vector: [ 1, 1, 1, 1 ] + num_candidates: 40 + + - match: { hits.total.value: 40 } + - length: { hits.hits: 5 } + - contains: {hits.hits: {_id: "1"}} + - contains: {hits.hits: {_id: "11"}} + - contains: {hits.hits: {_id: "2"}} + - contains: {hits.hits: {_id: "21"}} + - contains: {hits.hits: {_id: "31"}} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java index b89cea7dff089..c4737468a766c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java @@ -105,7 +105,9 @@ public TestAction( actionFilters, Request::new, Request::new, - threadPool.executor(ThreadPool.Names.GENERIC) + threadPool.executor(ThreadPool.Names.GENERIC), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java new file mode 100644 index 0000000000000..2490eade46d31 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESIntegTestCase; + +@ESIntegTestCase.ClusterScope(numDataNodes = 0) +public class ClusterRebalanceAllocationDeciderIT extends ESIntegTestCase { + public void testDefault() { + internalCluster().startNode(); + assertEquals( + ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS, + ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.get( + internalCluster().getInstance(ClusterService.class).getSettings() + ) + ); + } + + public void testDefaultLegacyAllocator() { + internalCluster().startNode( + Settings.builder().put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), ClusterModule.BALANCED_ALLOCATOR) + ); + assertEquals( + ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_ALL_ACTIVE, + ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.get( + internalCluster().getInstance(ClusterService.class).getSettings() + ) + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 384a004861776..213c8003b7047 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -53,14 +53,12 @@ import org.elasticsearch.transport.TransportService; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; @@ -78,6 +76,36 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction> allSnapshotInfos = ConcurrentCollections.newQueue(); + private final List allSnapshotInfos = Collections.synchronizedList(new ArrayList<>()); /** * Accumulates number of snapshots that match the name/fromSortValue/slmPolicy predicates, to be returned in the response. */ private final AtomicInteger totalCount = new AtomicInteger(); - /** - * Accumulates the number of snapshots that match the name/fromSortValue/slmPolicy/after predicates, for sizing the final result - * list. - */ - private final AtomicInteger resultsCount = new AtomicInteger(); - GetSnapshotsOperation( CancellableTask cancellableTask, List repositories, @@ -269,7 +286,14 @@ void getMultipleReposSnapshotInfo(ActionListener listener) } }) - .addListener(listener.map(ignored -> buildResponse()), executor, threadPool.getThreadContext()); + .addListener( + listener.map(ignored -> buildResponse()), + // If we didn't load any SnapshotInfo blobs from the repo (e.g. verbose=false or current-snapshots-only) then this + // listener chain will already be complete, no need to fork again. Otherwise we forked to SNAPSHOT_META so must + // fork back to MANAGEMENT for the final step. + executor, + threadPool.getThreadContext() + ); } private void maybeGetRepositoryData(String repositoryName, ActionListener listener) { @@ -334,6 +358,7 @@ private AsyncSnapshotInfo forSnapshotInProgress(SnapshotsInProgress.Entry snapsh return new AsyncSnapshotInfo() { @Override public void getSnapshotInfo(ActionListener listener) { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // see [NOTE ON THREADING] final var snapshotInfo = SnapshotInfo.inProgress(snapshotInProgress); listener.onResponse(verbose ? snapshotInfo : snapshotInfo.basic()); } @@ -358,8 +383,11 @@ private AsyncSnapshotInfo forCompletedSnapshot( @Override public void getSnapshotInfo(ActionListener listener) { if (verbose) { + // always forks to SNAPSHOT_META, and may already have done so for an earlier item - see [NOTE ON THREADING] + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT, ThreadPool.Names.SNAPSHOT_META); getSnapshotInfoExecutor.getSnapshotInfo(repository, snapshotId, listener); } else { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // see [NOTE ON THREADING] ActionListener.completeWith( listener, () -> new SnapshotInfo( @@ -402,9 +430,11 @@ private Iterator getAsyncSnapshotInfoIterator(Repository repo this::forSnapshotInProgress ), repositoryData == null - // only returning in-progress snapshots + // Only returning in-progress snapshots: ? Collections.emptyIterator() - // also return matching completed snapshots (except any ones that were also found to be in-progress) + // Also return matching completed snapshots (except any ones that were also found to be in-progress). + // NB this will fork tasks to SNAPSHOT_META (if verbose=true) which will be used for subsequent items so we mustn't + // follow it with any more non-forking iteration. See [NOTE ON THREADING]. : Iterators.map( Iterators.filter( repositoryData.getSnapshotIds().iterator(), @@ -438,18 +468,7 @@ private void loadSnapshotInfos(Iterator asyncSnapshotInfoIter if (cancellableTask.notifyIfCancelled(listener)) { return; } - final var repositoryTotalCount = new AtomicInteger(); - - final List snapshots = new ArrayList<>(); - final List syncSnapshots = Collections.synchronizedList(snapshots); - try (var listeners = new RefCountingListener(listener)) { - final var iterationCompleteListener = listeners.acquire(ignored -> { - totalCount.addAndGet(repositoryTotalCount.get()); - // no need to synchronize access to snapshots: all writes happen-before this read - resultsCount.addAndGet(snapshots.size()); - allSnapshotInfos.add(snapshots); - }); ThrottledIterator.run( Iterators.failFast(asyncSnapshotInfoIterator, () -> cancellableTask.isCancelled() || listeners.isFailing()), (ref, asyncSnapshotInfo) -> { @@ -458,9 +477,9 @@ private void loadSnapshotInfos(Iterator asyncSnapshotInfoIter @Override public void onResponse(SnapshotInfo snapshotInfo) { if (matchesPredicates(snapshotInfo)) { - repositoryTotalCount.incrementAndGet(); + totalCount.incrementAndGet(); if (afterPredicate.test(snapshotInfo)) { - syncSnapshots.add(snapshotInfo.maybeWithoutIndices(indices)); + allSnapshotInfos.add(snapshotInfo.maybeWithoutIndices(indices)); } } refListener.onResponse(null); @@ -479,22 +498,21 @@ public void onFailure(Exception e) { }, getSnapshotInfoExecutor.getMaxRunningTasks(), () -> {}, - () -> iterationCompleteListener.onResponse(null) + () -> {} ); } } private GetSnapshotsResponse buildResponse() { - assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // see [NOTE ON THREADING] cancellableTask.ensureNotCancelled(); int remaining = 0; final var resultsStream = allSnapshotInfos.stream() - .flatMap(Collection::stream) .peek(this::assertSatisfiesAllPredicates) .sorted(sortBy.getSnapshotInfoComparator(order)) .skip(offset); final List snapshotInfos; - if (size == GetSnapshotsRequest.NO_LIMIT || resultsCount.get() <= size) { + if (size == GetSnapshotsRequest.NO_LIMIT || allSnapshotInfos.size() <= size) { snapshotInfos = resultsStream.toList(); } else { snapshotInfos = new ArrayList<>(size); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java index ac2f437f7225a..643f92ec3378f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java @@ -66,7 +66,9 @@ public TransportVerifyShardBeforeCloseAction( actionFilters, ShardRequest::new, ShardRequest::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + threadPool.executor(ThreadPool.Names.MANAGEMENT), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 74ae53f7ac9de..69e1309b89aef 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -58,7 +58,9 @@ public TransportShardFlushAction( actionFilters, ShardFlushRequest::new, ShardFlushRequest::new, - threadPool.executor(ThreadPool.Names.FLUSH) + threadPool.executor(ThreadPool.Names.FLUSH), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); transportService.registerRequestHandler( PRE_SYNCED_FLUSH_ACTION_NAME, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java index 31e9f959f0fe7..e93b3983ee85b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java @@ -67,7 +67,9 @@ public TransportVerifyShardIndexBlockAction( actionFilters, ShardRequest::new, ShardRequest::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + threadPool.executor(ThreadPool.Names.MANAGEMENT), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index b3e6385e7099d..cc4edcf0efb81 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -68,7 +68,9 @@ public TransportShardRefreshAction( actionFilters, BasicReplicationRequest::new, ShardRefreshReplicaRequest::new, - threadPool.executor(ThreadPool.Names.REFRESH) + threadPool.executor(ThreadPool.Names.REFRESH), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); // registers the unpromotable version of shard refresh action new TransportUnpromotableShardRefreshAction(clusterService, transportService, shardStateAction, actionFilters, indicesService); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 67e7e3be72a02..fc9df7bbf73b9 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -115,7 +115,7 @@ public TransportShardBulkAction( BulkShardRequest::new, BulkShardRequest::new, ExecutorSelector.getWriteExecutorForShard(threadPool), - false, + PrimaryActionExecution.RejectOnOverload, indexingPressure, systemIndices ); diff --git a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java index 4684c990299f9..5a891f33480fa 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java @@ -71,7 +71,7 @@ public TransportResyncReplicationAction( ResyncReplicationRequest::new, ResyncReplicationRequest::new, ExecutorSelector.getWriteExecutorForShard(threadPool), - true, /* we should never reject resync because of thread pool capacity on primary */ + PrimaryActionExecution.Force, /* we should never reject resync because of thread pool capacity on primary */ indexingPressure, systemIndices ); diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index ac5b004886319..c2d7e173fd0bf 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -89,6 +89,34 @@ public abstract class TransportReplicationAction< ReplicaRequest extends ReplicationRequest, Response extends ReplicationResponse> extends TransportAction { + /** + * Execution of the primary action + */ + protected enum PrimaryActionExecution { + /** + * Is subject to usual queue length and indexing pressure checks + */ + RejectOnOverload, + /** + * Will be "forced" (bypassing queue length and indexing pressure checks) + */ + Force + } + + /** + * Global checkpoint behaviour + */ + protected enum SyncGlobalCheckpointAfterOperation { + /** + * Do not sync as part of this action + */ + DoNotSync, + /** + * Attempt to sync the global checkpoint to the replica(s) after success + */ + AttemptAfterSuccess + } + /** * The timeout for retrying replication requests. */ @@ -128,36 +156,6 @@ public abstract class TransportReplicationAction< private volatile TimeValue initialRetryBackoffBound; private volatile TimeValue retryTimeout; - protected TransportReplicationAction( - Settings settings, - String actionName, - TransportService transportService, - ClusterService clusterService, - IndicesService indicesService, - ThreadPool threadPool, - ShardStateAction shardStateAction, - ActionFilters actionFilters, - Writeable.Reader requestReader, - Writeable.Reader replicaRequestReader, - Executor executor - ) { - this( - settings, - actionName, - transportService, - clusterService, - indicesService, - threadPool, - shardStateAction, - actionFilters, - requestReader, - replicaRequestReader, - executor, - false, - false - ); - } - @SuppressWarnings("this-escape") protected TransportReplicationAction( Settings settings, @@ -171,10 +169,12 @@ protected TransportReplicationAction( Writeable.Reader requestReader, Writeable.Reader replicaRequestReader, Executor executor, - boolean syncGlobalCheckpointAfterOperation, - boolean forceExecutionOnPrimary + SyncGlobalCheckpointAfterOperation syncGlobalCheckpointAfterOperation, + PrimaryActionExecution primaryActionExecution ) { super(actionName, actionFilters, transportService.getTaskManager()); + assert syncGlobalCheckpointAfterOperation != null : "Must specify global checkpoint sync behaviour"; + assert primaryActionExecution != null : "Must specify primary action execution behaviour"; this.threadPool = threadPool; this.transportService = transportService; this.clusterService = clusterService; @@ -187,7 +187,10 @@ protected TransportReplicationAction( this.initialRetryBackoffBound = REPLICATION_INITIAL_RETRY_BACKOFF_BOUND.get(settings); this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings); - this.forceExecutionOnPrimary = forceExecutionOnPrimary; + this.forceExecutionOnPrimary = switch (primaryActionExecution) { + case Force -> true; + case RejectOnOverload -> false; + }; transportService.registerRequestHandler( actionName, @@ -217,7 +220,10 @@ protected TransportReplicationAction( this.transportOptions = transportOptions(); - this.syncGlobalCheckpointAfterOperation = syncGlobalCheckpointAfterOperation; + this.syncGlobalCheckpointAfterOperation = switch (syncGlobalCheckpointAfterOperation) { + case AttemptAfterSuccess -> true; + case DoNotSync -> false; + }; ClusterSettings clusterSettings = clusterService.getClusterSettings(); clusterSettings.addSettingsUpdateConsumer(REPLICATION_INITIAL_RETRY_BACKOFF_BOUND, (v) -> initialRetryBackoffBound = v); diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java index 8994b428adcbe..f380710cc0794 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java @@ -76,7 +76,7 @@ protected TransportWriteAction( Writeable.Reader request, Writeable.Reader replicaRequest, BiFunction executorFunction, - boolean forceExecutionOnPrimary, + PrimaryActionExecution primaryActionExecution, IndexingPressure indexingPressure, SystemIndices systemIndices ) { @@ -94,8 +94,8 @@ protected TransportWriteAction( request, replicaRequest, EsExecutors.DIRECT_EXECUTOR_SERVICE, - true, - forceExecutionOnPrimary + SyncGlobalCheckpointAfterOperation.AttemptAfterSuccess, + primaryActionExecution ); this.executorFunction = executorFunction; this.indexingPressure = indexingPressure; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java index 88d4a652a5a39..7289b218b6be4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -44,7 +45,9 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider { private static final String CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE = "cluster.routing.allocation.allow_rebalance"; public static final Setting CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING = new Setting<>( CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, - ClusterRebalanceType.INDICES_ALL_ACTIVE.toString(), + settings -> ClusterModule.DESIRED_BALANCE_ALLOCATOR.equals(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.get(settings)) + ? ClusterRebalanceType.ALWAYS.toString() + : ClusterRebalanceType.INDICES_ALL_ACTIVE.toString(), ClusterRebalanceType::parseString, Property.Dynamic, Property.NodeScope diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index a0420fdc5e0ff..1fcaf1394d4a7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.mapper.DocumentDimensions; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; @@ -349,6 +350,10 @@ protected static String tsdbMode() { .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH) .field("type", DateFieldMapper.CONTENT_TYPE) .endObject() + .startObject("host.name") + .field("type", KeywordFieldMapper.CONTENT_TYPE) + .field("ignore_above", 1024) + .endObject() .endObject() .endObject()) ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index d27c0acdb6b2e..2c1ac0d35c898 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1318,7 +1318,9 @@ boolean supportsElementType(ElementType elementType) { boolean updatableTo(IndexOptions update) { return update.type.equals(this.type) || update.type.equals(VectorIndexType.HNSW.name) - || update.type.equals(VectorIndexType.INT8_HNSW.name); + || update.type.equals(VectorIndexType.INT8_HNSW.name) + || update.type.equals(VectorIndexType.INT4_HNSW.name) + || update.type.equals(VectorIndexType.INT4_FLAT.name); } } @@ -1425,7 +1427,14 @@ boolean supportsElementType(ElementType elementType) { @Override boolean updatableTo(IndexOptions update) { - return Objects.equals(this, update); + boolean updatable = update.type.equals(this.type); + if (updatable) { + Int4HnswIndexOptions int4HnswIndexOptions = (Int4HnswIndexOptions) update; + // fewer connections would break assumptions on max number of connections (based on largest previous graph) during merge + // quantization could not behave as expected with different confidence intervals (and quantiles) to be created + updatable = int4HnswIndexOptions.m >= this.m && confidenceInterval == int4HnswIndexOptions.confidenceInterval; + } + return updatable; } @Override @@ -1487,7 +1496,10 @@ boolean supportsElementType(ElementType elementType) { @Override boolean updatableTo(IndexOptions update) { // TODO: add support for updating from flat, hnsw, and int8_hnsw and updating params - return Objects.equals(this, update); + return update.type.equals(this.type) + || update.type.equals(VectorIndexType.HNSW.name) + || update.type.equals(VectorIndexType.INT8_HNSW.name) + || update.type.equals(VectorIndexType.INT4_HNSW.name); } @Override @@ -1562,8 +1574,8 @@ boolean supportsElementType(ElementType elementType) { @Override boolean updatableTo(IndexOptions update) { - boolean updatable = update.type.equals(this.type); - if (updatable) { + boolean updatable; + if (update.type.equals(this.type)) { Int8HnswIndexOptions int8HnswIndexOptions = (Int8HnswIndexOptions) update; // fewer connections would break assumptions on max number of connections (based on largest previous graph) during merge // quantization could not behave as expected with different confidence intervals (and quantiles) to be created @@ -1571,6 +1583,8 @@ boolean updatableTo(IndexOptions update) { updatable &= confidenceInterval == null || int8HnswIndexOptions.confidenceInterval != null && confidenceInterval.equals(int8HnswIndexOptions.confidenceInterval); + } else { + updatable = update.type.equals(VectorIndexType.INT4_HNSW.name) && ((Int4HnswIndexOptions) update).m >= this.m; } return updatable; } @@ -1602,7 +1616,9 @@ boolean updatableTo(IndexOptions update) { HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) update; updatable = hnswIndexOptions.m >= this.m; } - return updatable || (update.type.equals(VectorIndexType.INT8_HNSW.name) && ((Int8HnswIndexOptions) update).m >= m); + return updatable + || (update.type.equals(VectorIndexType.INT8_HNSW.name) && ((Int8HnswIndexOptions) update).m >= m) + || (update.type.equals(VectorIndexType.INT4_HNSW.name) && ((Int4HnswIndexOptions) update).m >= m); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java index 7d3df2c174a83..a051d9c2df430 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java @@ -63,8 +63,8 @@ public GlobalCheckpointSyncAction( Request::new, Request::new, threadPool.executor(ThreadPool.Names.WRITE), - false, - true + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.Force ); } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index 541e279d4cfbb..0aa0f0b8d1556 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -81,7 +81,9 @@ public RetentionLeaseBackgroundSyncAction( actionFilters, Request::new, Request::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + threadPool.executor(ThreadPool.Names.MANAGEMENT), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java index b5fe27fb20bc3..0efcf8ac9298b 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java @@ -91,7 +91,7 @@ public RetentionLeaseSyncAction( RetentionLeaseSyncAction.Request::new, RetentionLeaseSyncAction.Request::new, new ManagementOnlyExecutorFunction(threadPool), - false, + PrimaryActionExecution.RejectOnOverload, indexingPressure, systemIndices ); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index fcbddb581946b..04ad7d410e9b0 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -144,7 +144,7 @@ public static > R resolveRequest(TransportReques private static ThreadPool threadPool; - private boolean forceExecute; + private TransportReplicationAction.PrimaryActionExecution primaryActionExecution; private ClusterService clusterService; private TransportService transportService; private CapturingTransport transport; @@ -165,7 +165,7 @@ public static void beforeClass() { @Before public void setUp() throws Exception { super.setUp(); - forceExecute = randomBoolean(); + primaryActionExecution = randomFrom(TransportReplicationAction.PrimaryActionExecution.values()); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); transportService = transport.createTransportService( @@ -951,7 +951,7 @@ public void testSeqNoIsSetOnPrimary() { ActionListener argument = (ActionListener) invocation.getArguments()[0]; argument.onResponse(count::decrementAndGet); return null; - }).when(shard).acquirePrimaryOperationPermit(any(), any(Executor.class), eq(forceExecute)); + }).when(shard).acquirePrimaryOperationPermit(any(), any(Executor.class), eq(shouldForceAcquirePermit(primaryActionExecution))); when(shard.getActiveOperationsCount()).thenAnswer(i -> count.get()); final IndexService indexService = mock(IndexService.class); @@ -979,6 +979,13 @@ public void testSeqNoIsSetOnPrimary() { assertThat(shardRequest.getPrimaryTerm(), equalTo(primaryTerm)); } + private boolean shouldForceAcquirePermit(TransportReplicationAction.PrimaryActionExecution primaryActionExecution) { + return switch (primaryActionExecution) { + case Force -> true; + case RejectOnOverload -> false; + }; + } + public void testCounterOnPrimary() throws Exception { final String index = "test"; final ShardId shardId = new ShardId(index, "_na_", 0); @@ -1511,8 +1518,8 @@ private class TestAction extends TransportReplicationAction isPrimaryMode.get()); doAnswer(invocation -> { long term = (Long) invocation.getArguments()[0]; diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index d0ae26f97917a..c5642fd9681ac 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -456,7 +456,9 @@ private abstract class TestAction extends TransportReplicationAction()), Request::new, Request::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); this.shardId = Objects.requireNonNull(shardId); this.primary = Objects.requireNonNull(primary); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 37f02035a5f43..5cc0e55942818 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -426,7 +426,7 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF TestRequest::new, TestRequest::new, (service, ignore) -> EsExecutors.DIRECT_EXECUTOR_SERVICE, - false, + PrimaryActionExecution.RejectOnOverload, new IndexingPressure(Settings.EMPTY), EmptySystemIndices.INSTANCE ); @@ -454,7 +454,7 @@ protected TestAction( TestRequest::new, TestRequest::new, (service, ignore) -> EsExecutors.DIRECT_EXECUTOR_SERVICE, - false, + PrimaryActionExecution.RejectOnOverload, new IndexingPressure(settings), EmptySystemIndices.INSTANCE ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java index 328777bfe28e7..7f9c69955adcd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java @@ -583,20 +583,28 @@ public void testClusterAllActive3() { public void testRebalanceWithIgnoredUnassignedShards() { final AtomicBoolean allocateTest1 = new AtomicBoolean(false); - AllocationService strategy = createAllocationService(Settings.EMPTY, new TestGatewayAllocator() { - @Override - public void allocateUnassigned( - ShardRouting shardRouting, - RoutingAllocation allocation, - UnassignedAllocationHandler unassignedAllocationHandler - ) { - if (allocateTest1.get() == false && "test1".equals(shardRouting.index().getName())) { - unassignedAllocationHandler.removeAndIgnore(UnassignedInfo.AllocationStatus.NO_ATTEMPT, allocation.changes()); - } else { - super.allocateUnassigned(shardRouting, allocation, unassignedAllocationHandler); + AllocationService strategy = createAllocationService( + Settings.builder() + .put( + ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), + ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_ALL_ACTIVE.toString() + ) + .build(), + new TestGatewayAllocator() { + @Override + public void allocateUnassigned( + ShardRouting shardRouting, + RoutingAllocation allocation, + UnassignedAllocationHandler unassignedAllocationHandler + ) { + if (allocateTest1.get() == false && "test1".equals(shardRouting.index().getName())) { + unassignedAllocationHandler.removeAndIgnore(UnassignedInfo.AllocationStatus.NO_ATTEMPT, allocation.changes()); + } else { + super.allocateUnassigned(shardRouting, allocation, unassignedAllocationHandler); + } } } - }); + ); Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(0)) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 3dd4e31b9ca3f..4727c65be60e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -195,6 +195,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .field("element_type", "bit") ) ); + // update for flat checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -210,6 +211,21 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_flat\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_flat\"")) + ); checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -240,6 +256,22 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + // update for int8_flat checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -270,6 +302,56 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_flat\"")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + // update for hnsw checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -285,6 +367,37 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 100) + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"hnsw\"")) + ); checker.registerConflictCheck( "index_options", fieldMapping( @@ -304,6 +417,438 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject() ) ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + // update for int8_hnsw + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 256) + .endObject(), + m -> assertTrue(m.toString().contains("\"m\":256")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 256) + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); + // update for int4_flat + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"hnsw\"")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + // update for int4_hnsw + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("m", 256) + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"m\":256")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("confidence_interval", 0.03) + .field("m", 4) + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("confidence_interval", 0.03) + .field("m", 100) + .endObject(), + m -> assertTrue(m.toString().contains("\"m\":100")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("confidence_interval", 0.3) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); } @Override diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java index d9592c3df4950..2d0c43315f746 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java @@ -67,7 +67,7 @@ public TransportBulkShardOperationsAction( BulkShardOperationsRequest::new, BulkShardOperationsRequest::new, ExecutorSelector.getWriteExecutorForShard(threadPool), - false, + PrimaryActionExecution.RejectOnOverload, indexingPressure, systemIndices ); diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json deleted file mode 100644 index 167efbd3ffaf5..0000000000000 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "template": { - "mappings": { - "date_detection": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword" - }, - "data_stream.type": { - "type": "constant_keyword", - "value": "logs" - }, - "data_stream.dataset": { - "type": "constant_keyword" - }, - "data_stream.namespace": { - "type": "constant_keyword" - } - } - } - }, - "_meta": { - "description": "default mappings for the logs index template installed by x-pack", - "managed": true - }, - "version": ${xpack.stack.template.version}, - "deprecated": ${xpack.stack.template.deprecated} -} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 648146ccdcc61..7dc1dfb6cf3df 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -146,7 +146,7 @@ private Map loadComponentTemplateConfigs(boolean logs ), new IndexTemplateConfig( LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - logsDbEnabled ? "/logs@mappings-logsdb.json" : "/logs@mappings.json", + "/logs@mappings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES