diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java index 1667d674870e8..60ae306495800 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java @@ -62,10 +62,10 @@ static Provider registerTask(Project project, SourceSet sourc project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> { Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); testTask.dependsOn(bundle); - if (project.getPath().contains("modules:")) { + if (project.getPath().contains("modules:") || project.getPath().startsWith(":x-pack:plugin")) { testTask.getClusters().forEach(c -> c.module(bundle.getArchiveFile())); } else { - testTask.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))); + testTask.getClusters().forEach(c -> c.plugin(bundle.getArchiveFile())); } }); }); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index c946ea0db7872..327e9a753f8bb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -617,27 +617,23 @@ private void copyExtraJars() { } private void installModules() { - if (testDistribution == TestDistribution.INTEG_TEST) { - logToProcessStdout("Installing " + modules.size() + "modules"); - for (Provider module : modules) { - Path destination = getDistroDir().resolve("modules") - .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); - // only install modules that are not already bundled with the integ-test distribution - if (Files.exists(destination) == false) { - fileSystemOperations.copy(spec -> { - if (module.get().getName().toLowerCase().endsWith(".zip")) { - spec.from(archiveOperations.zipTree(module)); - } else if (module.get().isDirectory()) { - spec.from(module); - } else { - throw new IllegalArgumentException("Not a valid module " + module + " for " + this); - } - spec.into(destination); - }); - } + logToProcessStdout("Installing " + modules.size() + "modules"); + for (Provider module : modules) { + Path destination = getDistroDir().resolve("modules") + .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); + // only install modules that are not already bundled with the integ-test distribution + if (Files.exists(destination) == false) { + fileSystemOperations.copy(spec -> { + if (module.get().getName().toLowerCase().endsWith(".zip")) { + spec.from(archiveOperations.zipTree(module)); + } else if (module.get().isDirectory()) { + spec.from(module); + } else { + throw new IllegalArgumentException("Not a valid module " + module + " for " + this); + } + spec.into(destination); + }); } - } else { - LOGGER.info("Not installing " + modules.size() + "(s) since the " + distributions + " distribution already has them"); } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 29f6036195b87..5d74ea0de610b 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -335,7 +335,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { dependencies { libs project(':server') - libs project(':libs:elasticsearch-plugin-classloader') libs project(':distribution:tools:java-version-checker') libs project(':distribution:tools:launchers') diff --git a/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc b/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc index 3edb1080611d2..55d3818a3462a 100644 --- a/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc +++ b/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc @@ -30,7 +30,7 @@ analysis chain matches a predicate. `token.type` (`String`, read-only):: The type of the current token -`token.keyword` ('boolean`, read-only):: +`token.keyword` (`boolean`, read-only):: Whether or not the current token is marked as a keyword *Return* @@ -40,4 +40,4 @@ analysis chain matches a predicate. *API* -The standard <> is available. \ No newline at end of file +The standard <> is available. diff --git a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc index b52d9ece59742..5f06c85587c78 100644 --- a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc @@ -61,3 +61,20 @@ The API returns the following result: "acknowledged": true } -------------------------------------------------- + +This example deletes all autoscaling policies. + +[source,console] +-------------------------------------------------- +DELETE /_autoscaling/policy/* +-------------------------------------------------- +// TEST + +The API returns the following result: + +[source,console-result] +-------------------------------------------------- +{ + "acknowledged": true +} +-------------------------------------------------- diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc index f9c9991a60bb1..7c8bdef996b78 100644 --- a/docs/reference/cat/indices.asciidoc +++ b/docs/reference/cat/indices.asciidoc @@ -77,7 +77,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segme `local`:: (Optional, boolean) + -deprecated::[7.10.0,"This parameter does not affect the request. It will be removed in a future release."] +deprecated::[7.11.0,"This parameter does not affect the request. It will be removed in a future release."] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index 46adb53d3222c..7a69cd4da5cce 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -276,7 +276,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] `local`:: (Optional, boolean) + -deprecated::[7.10.0,"This parameter does not affect the request. It will be removed in a future release."] +deprecated::[7.11.0,"This parameter does not affect the request. It will be removed in a future release."] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] diff --git a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc index 7b4e24057c655..ea5aa3c567806 100644 --- a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc +++ b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc @@ -19,7 +19,7 @@ for data tier filtering. [discrete] [[data-tier-allocation-filters]] -====Data tier allocation settings +==== Data tier allocation settings `index.routing.allocation.include._tier`:: @@ -43,7 +43,6 @@ for data tier filtering. Assign the index to the first tier in the list that has an available node. This prevents indices from remaining unallocated if no nodes are available in the preferred tier. - For example, if you set `index.routing.allocation.include._tier_preference` to `data_warm,data_hot`, the index is allocated to the warm tier if there are nodes with the `data_warm` role. If there are no nodes in the warm tier, diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index d6446bec5e5aa..9f8ded246243a 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -5,7 +5,7 @@ ++++ Creates or updates a component template. -Component templates are building blocks for constructing <>. +Component templates are building blocks for constructing <> that specify index <>, <>, and <>. diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc index 8039d9648b81b..77b5cf9b9063a 100644 --- a/docs/reference/ingest/apis/get-pipeline.asciidoc +++ b/docs/reference/ingest/apis/get-pipeline.asciidoc @@ -45,8 +45,12 @@ GET /_ingest/pipeline/my-pipeline-id [[get-pipeline-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=path-pipeline] - +``:: +(Optional, string) +Comma-separated list of pipeline IDs to retrieve. Wildcard (`*`) expressions are +supported. ++ +To get all ingest pipelines, omit this parameter or use `*`. [[get-pipeline-api-query-params]] diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index a55b8f23e9c23..d1062e79bb588 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -277,14 +277,6 @@ To create a dedicated ingest node, set: node.roles: [ ingest ] ---- -[[node-ingest-node-setting]] -// tag::node-ingest-tag[] -`node.ingest` {ess-icon}:: -Determines whether a node is an ingest node. <> can apply -an ingest pipeline to transform and enrich a document before indexing. Default: -`true`. -// end::node-ingest-tag[] - [[coordinating-only-node]] ==== Coordinating only node diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index ae7209becfab9..677c3be00dab6 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -642,12 +642,6 @@ The number of search or bulk index operations processed. Documents are processed in batches instead of individually. end::pages-processed[] -tag::path-pipeline[] -``:: -(Optional, string) Comma-separated list or wildcard expression of pipeline IDs -used to limit the request. -end::path-pipeline[] - tag::pivot[] The method for transforming the data. These objects define the pivot function `group by` fields and the aggregation to reduce the data. diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc index c84c6e41cd9a8..e7428b54892f0 100644 --- a/docs/reference/search/suggesters/phrase-suggest.asciidoc +++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc @@ -149,7 +149,7 @@ The response contains suggestions scored by the most likely spelling correction `gram_size` is set to the `max_shingle_size` if not explicitly set. `real_word_error_likelihood`:: - The likelihood of a term being a + The likelihood of a term being misspelled even if the term exists in the dictionary. The default is `0.95`, meaning 5% of the real words are misspelled. diff --git a/docs/reference/setup/important-settings/path-settings.asciidoc b/docs/reference/setup/important-settings/path-settings.asciidoc index 2867c464e4824..49e6edb356bb0 100644 --- a/docs/reference/setup/important-settings/path-settings.asciidoc +++ b/docs/reference/setup/important-settings/path-settings.asciidoc @@ -2,32 +2,32 @@ [discrete] === Path settings -If you are using the `.zip` or `.tar.gz` archives, the `data` and `logs` -directories are sub-folders of `$ES_HOME`. If these important folders are left -in their default locations, there is a high risk of them being deleted while -upgrading {es} to a new version. - -In production use, you will almost certainly want to change the locations of the -`path.data` and `path.logs` folders: - -[source,yaml] --------------------------------------------------- -path: - logs: /var/log/elasticsearch - data: /var/data/elasticsearch --------------------------------------------------- - -The RPM and Debian distributions already use custom paths for `data` and `logs`. - -The `path.data` settings can be set to multiple paths, in which case all paths -will be used to store data. However, the files belonging to a single shard will -all be stored on the same data path: - -[source,yaml] --------------------------------------------------- -path: - data: - - /mnt/elasticsearch_1 - - /mnt/elasticsearch_2 - - /mnt/elasticsearch_3 --------------------------------------------------- +For <>, <>, and +<> installations, {es} writes data and logs to the +respective `data` and `logs` subdirectories of `$ES_HOME` by default. +However, files in `$ES_HOME` risk deletion during an upgrade. + +In production, we strongly recommend you set the `path.data` and `path.logs` in +`elasticsearch.yml` to locations outside of `$ES_HOME`. + +TIP: <>, <>, <>, <>, +and <> installations write data and log to locations +outside of `$ES_HOME` by default. + +Supported `path.data` and `path.logs` values vary by platform: + +include::{es-repo-dir}/tab-widgets/code.asciidoc[] + +include::{es-repo-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[] + +If needed, you can specify multiple paths in `path.data`. {es} stores the node's +data across all provided paths but keeps each shard's data on the same path. + +WARNING: {es} does not balance shards across a node's data paths. High disk +usage in a single path can trigger a <> for the entire node. If triggered, {es} will not add shards to +the node, even if the node’s other paths have available disk space. If you need +additional disk space, we recommend you add a new node rather than additional +data paths. + +include::{es-repo-dir}/tab-widgets/multi-data-path-widget.asciidoc[] \ No newline at end of file diff --git a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc index a1b180980ca3a..1fcd91d6c8c18 100644 --- a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc @@ -150,11 +150,11 @@ plugins: + -- (Required, object) -Contains settings for the repository. Valid properties for the `settings` object -depend on the repository type, set using the -<> parameter. +Contains settings for the repository. -.Valid `settings` properties for `fs` repositories +The following `settings` properties are valid for all repository types: + +.Properties of `settings` [%collapsible%open] ==== `chunk_size`:: @@ -168,11 +168,13 @@ file size). If `true`, metadata files, such as index mappings and settings, are compressed in snapshots. Data files are not compressed. Defaults to `true`. -`location`:: -(Required, string) -Location of the shared filesystem used to store and retrieve snapshots. This -location must be registered in the `path.repo` setting on all master and data -nodes in the cluster. +`max_number_of_snapshots`:: +(Optional, integer) +Maximum number of snapshots the repository can contain. Defaults to `500`. ++ +WARNING: We do not recommend increasing `max_number_of_snapshots`. Larger +snapshot repositories may degrade master node performance and cause stability +issues. Instead, delete older snapshots or use multiple repositories. `max_restore_bytes_per_sec`:: (Optional, <>) @@ -206,6 +208,19 @@ the repository but not create snapshots in it. ===== ==== +Other accepted `settings` properties depend on the repository type, set using the +<> parameter. + +.Valid `settings` properties for `fs` repositories +[%collapsible%open] +==== +`location`:: +(Required, string) +Location of the shared filesystem used to store and retrieve snapshots. This +location must be registered in the `path.repo` setting on all master and data +nodes in the cluster. +==== + .Valid `settings` properties for `source` repositories [%collapsible%open] ==== diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc index f0ae042cb349c..42a0f10c8bbd2 100644 --- a/docs/reference/snapshot-restore/register-repository.asciidoc +++ b/docs/reference/snapshot-restore/register-repository.asciidoc @@ -100,71 +100,20 @@ are left untouched and in place. [[snapshots-filesystem-repository]] === Shared file system repository -The shared file system repository (`"type": "fs"`) uses the shared file system to store snapshots. In order to register -the shared file system repository it is necessary to mount the same shared filesystem to the same location on all -master and data nodes. This location (or one of its parent directories) must be registered in the `path.repo` -setting on all master and data nodes. +Use a shared file system repository (`"type": "fs"`) to store snapshots on a +shared file system. -Assuming that the shared filesystem is mounted to `/mount/backups/my_fs_backup_location`, the following setting should -be added to `elasticsearch.yml` file: +To register a shared file system repository, first mount the file system to the +same location on all master and data nodes. Then add the file system's +path or parent directory to the `path.repo` setting in `elasticsearch.yml` for +each master and data node. For running clusters, this requires a +<> of each node. -[source,yaml] --------------- -path.repo: ["/mount/backups", "/mount/longterm_backups"] --------------- - -The `path.repo` setting supports Microsoft Windows UNC paths as long as at least server name and share are specified as -a prefix and back slashes are properly escaped: - -[source,yaml] --------------- -path.repo: ["\\\\MY_SERVER\\Snapshots"] --------------- - -After all nodes are restarted, the following command can be used to register the shared file system repository with -the name `my_fs_backup`: - -[source,console] ------------------------------------ -PUT /_snapshot/my_fs_backup -{ - "type": "fs", - "settings": { - "location": "/mount/backups/my_fs_backup_location", - "compress": true - } -} ------------------------------------ -// TEST[skip:no access to absolute path] - -If the repository location is specified as a relative path this path will be resolved against the first path specified -in `path.repo`: +Supported `path.repo` values vary by platform: -[source,console] ------------------------------------ -PUT /_snapshot/my_fs_backup -{ - "type": "fs", - "settings": { - "location": "my_fs_backup_location", - "compress": true - } -} ------------------------------------ -// TEST[continued] +include::{es-repo-dir}/tab-widgets/code.asciidoc[] -The following settings are supported: - -`location`:: Location of the snapshots. Mandatory. -`compress`:: Turns on compression of the snapshot files. Compression is applied only to metadata files (index mapping and settings). Data files are not compressed. Defaults to `true`. -`chunk_size`:: Big files can be broken down into chunks during snapshotting if needed. Specify the chunk size as a value and -unit, for example: `1GB`, `10MB`, `5KB`, `500B`. Defaults to `null` (unlimited chunk size). -`max_restore_bytes_per_sec`:: Throttles per node restore rate. Defaults to unlimited. Note that restores are also throttled through <>. -`max_snapshot_bytes_per_sec`:: Throttles per node snapshot rate. Defaults to `40mb` per second. -`readonly`:: Makes repository read-only. Defaults to `false`. -`max_number_of_snapshots`:: Limits the maximum number of snapshots that the repository may contain. Defaults to `500`. Note that snapshot repositories do not -scale indefinitely in size and might lead to master node performance and stability issues if they grow past a certain size. We do not recommend increasing this setting. -Instead you should delete older snapshots or use multiple repositories. +include::{es-repo-dir}/tab-widgets/register-fs-repo-widget.asciidoc[] [discrete] [[snapshots-read-only-repository]] diff --git a/docs/reference/tab-widgets/customize-data-log-path-widget.asciidoc b/docs/reference/tab-widgets/customize-data-log-path-widget.asciidoc new file mode 100644 index 0000000000000..a87f4c6d5576d --- /dev/null +++ b/docs/reference/tab-widgets/customize-data-log-path-widget.asciidoc @@ -0,0 +1,39 @@ +++++ +
+
+ + +
+
+++++ + +include::customize-data-log-path.asciidoc[tag=unix] + +++++ +
+ +
+++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/customize-data-log-path.asciidoc b/docs/reference/tab-widgets/customize-data-log-path.asciidoc new file mode 100644 index 0000000000000..45cee2babeb29 --- /dev/null +++ b/docs/reference/tab-widgets/customize-data-log-path.asciidoc @@ -0,0 +1,22 @@ +// tag::unix[] +Linux and macOS installations support Unix-style paths: + +[source,yaml] +---- +path: + data: /var/data/elasticsearch + logs: /var/log/elasticsearch +---- +// end::unix[] + + +// tag::win[] +Windows installations support DOS paths with escaped backslashes: + +[source,yaml] +---- +path: + data: "C:\\Elastic\\Elasticsearch\\data" + logs: "C:\\Elastic\\Elasticsearch\\logs" +---- +// end::win[] \ No newline at end of file diff --git a/docs/reference/tab-widgets/logging-widget.asciidoc b/docs/reference/tab-widgets/logging-widget.asciidoc index 8b20c525ca1e9..b88633bdb48a8 100644 --- a/docs/reference/tab-widgets/logging-widget.asciidoc +++ b/docs/reference/tab-widgets/logging-widget.asciidoc @@ -25,7 +25,7 @@ aria-controls="mac-tab-logs" id="mac-logs" tabindex="-1"> - MacOS + macOS + + +
+++++ + +include::multi-data-path.asciidoc[tag=unix] + +++++ +
+ + +++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/multi-data-path.asciidoc b/docs/reference/tab-widgets/multi-data-path.asciidoc new file mode 100644 index 0000000000000..0a63c7791f66c --- /dev/null +++ b/docs/reference/tab-widgets/multi-data-path.asciidoc @@ -0,0 +1,26 @@ +// tag::unix[] +Linux and macOS installations support multiple Unix-style paths in `path.data`: + +[source,yaml] +---- +path: + data: + - /mnt/elasticsearch_1 + - /mnt/elasticsearch_2 + - /mnt/elasticsearch_3 +---- +// end::unix[] + + +// tag::win[] +Windows installations support multiple DOS paths in `path.data`: + +[source,yaml] +---- +path: + data: + - "C:\\Elastic\\Elasticsearch_1" + - "E:\\Elastic\\Elasticsearch_1" + - "F:\\Elastic\\Elasticsearch_3" +---- +// end::win[] diff --git a/docs/reference/tab-widgets/register-fs-repo-widget.asciidoc b/docs/reference/tab-widgets/register-fs-repo-widget.asciidoc new file mode 100644 index 0000000000000..d0ac7041adf2b --- /dev/null +++ b/docs/reference/tab-widgets/register-fs-repo-widget.asciidoc @@ -0,0 +1,39 @@ +++++ +
+
+ + +
+
+++++ + +include::register-fs-repo.asciidoc[tag=unix] + +++++ +
+ +
+++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/register-fs-repo.asciidoc b/docs/reference/tab-widgets/register-fs-repo.asciidoc new file mode 100644 index 0000000000000..bbfeabcbf571c --- /dev/null +++ b/docs/reference/tab-widgets/register-fs-repo.asciidoc @@ -0,0 +1,103 @@ +// tag::unix[] +Linux and macOS installations support Unix-style paths: + +[source,yaml] +---- +path: + repo: + - /mount/backups + - /mount/long_term_backups +---- + +After restarting each node, use the <> API to register the file system repository. Specify the file +system's path in `settings.location`: + +[source,console] +---- +PUT /_snapshot/my_fs_backup +{ + "type": "fs", + "settings": { + "location": "/mount/backups/my_fs_backup_location", + "compress": true + } +} +---- +// TEST[skip:no access to path] + +If you specify a relative path in `settings.location`, {es} resolves the path +using the first value in the `path.repo` setting. + +[source,console] +---- +PUT /_snapshot/my_fs_backup +{ + "type": "fs", + "settings": { + "location": "my_fs_backup_location", <1> + "compress": true + } +} +---- +// TEST[skip:no access to path] + +<1> The first value in the `path.repo` setting is `/mount/backups`. This +relative path, `my_fs_backup_location`, resolves to +`/mount/backups/my_fs_backup_location`. +// end::unix[] + + +// tag::win[] +Windows installations support both DOS and Microsoft UNC paths. Escaped any +backslashes in the paths. For UNC paths, provide the server and share name as a +prefix. + +[source,yaml] +---- +path: + repo: + - "E:\\Mount\\Backups" <1> + - "\\\\MY_SERVER\\Mount\\Long_term_backups" <2> +---- + +<1> DOS path +<2> UNC path + +After restarting each node, use the <> API to register the file system repository. Specify the file +system's path in `settings.location`: + +[source,console] +---- +PUT /_snapshot/my_fs_backup +{ + "type": "fs", + "settings": { + "location": "E:\\Mount\\Backups\\My_fs_backup_location", + "compress": true + } +} +---- +// TEST[skip:no access to path] + +If you specify a relative path in `settings.location`, {es} resolves the path +using the first value in the `path.repo` setting. + +[source,console] +---- +PUT /_snapshot/my_fs_backup +{ + "type": "fs", + "settings": { + "location": "My_fs_backup_location", <1> + "compress": true + } +} +---- +// TEST[skip:no access to path] + +<1> The first value in the `path.repo` setting is `E:\Mount\Backups`. This +relative path, `My_fs_backup_location`, resolves to +`E:\Mount\Backups\My_fs_backup_location`. +// end::win[] \ No newline at end of file diff --git a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy index bcc5eef3193d7..56391a9ab5280 100644 --- a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy @@ -22,9 +22,6 @@ grant { // needed to apply additional sandboxing to tika parsing permission java.security.SecurityPermission "createAccessControlContext"; - // TODO: fix PDFBox not to actually install bouncy castle like this - permission java.security.SecurityPermission "putProviderProperty.BC"; - permission java.security.SecurityPermission "insertProvider"; // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597 permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; // needed by xmlbeans, as part of POI for MS xml docs diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json index 165bf1a3f235a..ed821ca52ed53 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json @@ -53,7 +53,7 @@ "type":"boolean", "description":"Return local information, do not retrieve the state from master node (default: false)", "deprecated":{ - "version":"8.0.0", + "version":"7.11.0", "description":"This parameter does not affect the request. It will be removed in a future release." } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json index fcf82e3e8ad82..576cba3fdcd49 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json @@ -53,7 +53,7 @@ "type":"boolean", "description":"Return local information, do not retrieve the state from master node (default: false)", "deprecated":{ - "version":"8.0.0", + "version":"7.11.0", "description":"This parameter does not affect the request. It will be removed in a future release." } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml index 2371bd5ef86ca..14a5c63862e94 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml @@ -495,6 +495,58 @@ setup: date: type: date + - do: + bulk: + index: test_2 + refresh: true + body: + - '{"index": {}}' + - '{"date": "2000-01-01"}' # This date is intenationally very far in the past so we end up not being able to use the date_histo -> range -> filters optimization + - '{"index": {}}' + - '{"date": "2000-01-02"}' + - '{"index": {}}' + - '{"date": "2016-02-01"}' + - '{"index": {}}' + - '{"date": "2016-03-01"}' + + - do: + search: + index: test_2 + body: + size: 0 + profile: true + aggs: + histo: + date_histogram: + field: date + calendar_interval: month + - match: { hits.total.value: 4 } + - length: { aggregations.histo.buckets: 195 } + - match: { aggregations.histo.buckets.0.key_as_string: "2000-01-01T00:00:00.000Z" } + - match: { aggregations.histo.buckets.0.doc_count: 2 } + - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator } + - match: { profile.shards.0.aggregations.0.description: histo } + - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 4 } + - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 } + +--- +"date_histogram run as filters profiler": + - skip: + version: " - 7.99.99" + reason: optimization added in 7.11.0, backport pending + + - do: + indices.create: + index: test_2 + body: + settings: + number_of_replicas: 0 + number_of_shards: 1 + mappings: + properties: + date: + type: date + - do: bulk: index: test_2 @@ -524,10 +576,13 @@ setup: - length: { aggregations.histo.buckets: 3 } - match: { aggregations.histo.buckets.0.key_as_string: "2016-01-01T00:00:00.000Z" } - match: { aggregations.histo.buckets.0.doc_count: 2 } - - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator } + - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator.FromDateRange } - match: { profile.shards.0.aggregations.0.description: histo } - - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 4 } - - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 } + # ultimately this ends up as a filters agg that uses filter by filter collection which is tracked in build_leaf_collector + - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 0 } + - match: { profile.shards.0.aggregations.0.debug.delegate: RangeAggregator.FromFilters } + - match: { profile.shards.0.aggregations.0.debug.delegate_debug.delegate: FiltersAggregator.FilterByFilter } + - match: { profile.shards.0.aggregations.0.debug.delegate_debug.delegate_debug.segments_with_deleted_docs: 0 } --- "histogram with hard bounds": diff --git a/server/build.gradle b/server/build.gradle index fb28a10e0dc84..ddf7e0420309c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -39,8 +39,7 @@ dependencies { api project(':libs:elasticsearch-x-content') api project(":libs:elasticsearch-geo") - compileOnly project(':libs:elasticsearch-plugin-classloader') - testRuntimeOnly project(':libs:elasticsearch-plugin-classloader') + implementation project(':libs:elasticsearch-plugin-classloader') // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java index cf0fecb6375f3..565061fb0d171 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java @@ -590,5 +590,10 @@ public ScoreMode scoreMode() { @Override public void preCollection() throws IOException {} + + @Override + public Aggregator[] subAggregators() { + throw new UnsupportedOperationException(); + } } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java index 9f324d1a5c8b5..11cb5aa77a47c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java @@ -37,6 +37,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; @@ -47,6 +49,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import static java.util.Collections.singleton; @@ -65,7 +68,17 @@ public class GetActionIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(InternalSettingsPlugin.class); + return List.of(InternalSettingsPlugin.class, SearcherWrapperPlugin.class); + } + + public static class SearcherWrapperPlugin extends Plugin { + @Override + public void onIndexModule(IndexModule indexModule) { + super.onIndexModule(indexModule); + if (randomBoolean()) { + indexModule.setReaderWrapper(indexService -> EngineTestCase.randomReaderWrapper()); + } + } } public void testSimpleGet() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index 0b6d12d59726a..dd85431a087ee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -38,10 +38,10 @@ import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; +import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; import org.elasticsearch.search.aggregations.metrics.Avg; import org.elasticsearch.search.aggregations.metrics.Sum; import org.elasticsearch.test.ESIntegTestCase; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index a91fe3c5c219e..f6881592a5ad7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -115,7 +115,7 @@ public ClusterState applyStartedShards(ClusterState clusterState, List searcherFactory, - SearcherScope scope) throws EngineException { - final Engine.Searcher searcher = searcherFactory.apply("get", scope); + protected final GetResult getFromSearcher(Get get, Engine.Searcher searcher) throws EngineException { final DocIdAndVersion docIdAndVersion; try { docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersion(searcher.getIndexReader(), get.uid(), true); @@ -596,7 +594,7 @@ protected final GetResult getFromSearcher(Get get, BiFunction searcherFactory) throws EngineException; + public abstract GetResult get(Get get, DocumentMapper mapper, Function searcherWrapper); /** * Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand. @@ -1616,6 +1614,7 @@ private GetResult(boolean exists, long version, DocIdAndVersion docIdAndVersion, this.docIdAndVersion = docIdAndVersion; this.searcher = searcher; this.fromTranslog = fromTranslog; + assert fromTranslog == false || searcher.getIndexReader() instanceof TranslogLeafReader; } public GetResult(Engine.Searcher searcher, DocIdAndVersion docIdAndVersion, boolean fromTranslog) { @@ -1630,6 +1629,10 @@ public long version() { return this.version; } + /** + * Returns {@code true} iff the get was performed from a translog operation. Notes that this returns {@code false} + * if the get was performed on an in-memory Lucene segment created from the corresponding translog operation. + */ public boolean isFromTranslog() { return fromTranslog; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index db5ac0db99da8..0daf3d1183f91 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -73,6 +73,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.fieldvisitor.IdOnlyFieldVisitor; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.ParseContext; @@ -618,14 +619,34 @@ private ExternalReaderManager createReaderManager(RefreshWarmerListener external } } + private GetResult getFromTranslog(Get get, Translog.Index index, DocumentMapper mapper, + Function searcherWrapper) throws IOException { + assert get.isReadFromTranslog(); + final SingleDocDirectoryReader inMemoryReader = new SingleDocDirectoryReader(shardId, index, mapper, config().getAnalyzer()); + final Engine.Searcher searcher = new Engine.Searcher("realtime_get", ElasticsearchDirectoryReader.wrap(inMemoryReader, shardId), + config().getSimilarity(), config().getQueryCache(), config().getQueryCachingPolicy(), inMemoryReader); + final Searcher wrappedSearcher = searcherWrapper.apply(searcher); + if (wrappedSearcher == searcher) { + searcher.close(); + assert inMemoryReader.assertMemorySegmentStatus(false); + final TranslogLeafReader translogLeafReader = new TranslogLeafReader(index); + return new GetResult(new Engine.Searcher("realtime_get", translogLeafReader, + IndexSearcher.getDefaultSimilarity(), null, IndexSearcher.getDefaultQueryCachingPolicy(), translogLeafReader), + new VersionsAndSeqNoResolver.DocIdAndVersion( + 0, index.version(), index.seqNo(), index.primaryTerm(), translogLeafReader, 0), true); + } else { + assert inMemoryReader.assertMemorySegmentStatus(true); + return getFromSearcher(get, wrappedSearcher); + } + } + @Override - public GetResult get(Get get, BiFunction searcherFactory) throws EngineException { + public GetResult get(Get get, DocumentMapper mapper, Function searcherWrapper) { assert Objects.equals(get.uid().field(), IdFieldMapper.NAME) : get.uid().field(); try (ReleasableLock ignored = readLock.acquire()) { ensureOpen(); - SearcherScope scope; if (get.realtime()) { - VersionValue versionValue = null; + final VersionValue versionValue; try (Releasable ignore = versionMap.acquireLock(get.uid().bytes())) { // we need to lock here to access the version map to do this truly in RT versionValue = getVersionFromMap(get.uid().bytes()); @@ -649,15 +670,9 @@ public GetResult get(Get get, BiFunction // the update call doesn't need the consistency since it's source only + _parent but parent can go away in 7.0 if (versionValue.getLocation() != null) { try { - Translog.Operation operation = translog.readOperation(versionValue.getLocation()); + final Translog.Operation operation = translog.readOperation(versionValue.getLocation()); if (operation != null) { - // in the case of a already pruned translog generation we might get null here - yet very unlikely - final Translog.Index index = (Translog.Index) operation; - TranslogLeafReader reader = new TranslogLeafReader(index); - return new GetResult(new Engine.Searcher("realtime_get", reader, - IndexSearcher.getDefaultSimilarity(), null, IndexSearcher.getDefaultQueryCachingPolicy(), reader), - new VersionsAndSeqNoResolver.DocIdAndVersion(0, index.version(), index.seqNo(), index.primaryTerm(), - reader, 0), true); + return getFromTranslog(get, (Translog.Index) operation, mapper, searcherWrapper); } } catch (IOException e) { maybeFailEngine("realtime_get", e); // lets check if the translog has failed with a tragic event @@ -670,14 +685,11 @@ public GetResult get(Get get, BiFunction assert versionValue.seqNo >= 0 : versionValue; refreshIfNeeded("realtime_get", versionValue.seqNo); } - scope = SearcherScope.INTERNAL; + return getFromSearcher(get, acquireSearcher("realtime_get", SearcherScope.INTERNAL, searcherWrapper)); } else { // we expose what has been externally expose in a point in time snapshot via an explicit refresh - scope = SearcherScope.EXTERNAL; + return getFromSearcher(get, acquireSearcher("get", SearcherScope.EXTERNAL, searcherWrapper)); } - - // no version, get the version from the index, we know that we refresh on flush - return getFromSearcher(get, searcherFactory, scope); } } @@ -1656,6 +1668,8 @@ final boolean refresh(String source, SearcherScope scope, boolean block) throws @Override public void writeIndexingBuffer() throws EngineException { + // TODO: revise https://github.com/elastic/elasticsearch/pull/34553 to use IndexWriter.flushNextBuffer to flush only the largest + // pending DWPT. Note that benchmarking this PR with a heavy update user case (geonames) and a small heap (1GB) caused OOM. refresh("write indexing buffer", SearcherScope.INTERNAL, false); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java index 4bc697e018ee4..c9d9baaef0d26 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -48,7 +49,6 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Stream; @@ -219,8 +219,8 @@ private static TranslogStats translogStats(final EngineConfig config, final Segm } @Override - public GetResult get(Get get, BiFunction searcherFactory) throws EngineException { - return getFromSearcher(get, searcherFactory, SearcherScope.EXTERNAL); + public GetResult get(Get get, DocumentMapper mapper, Function searcherWrapper) { + return getFromSearcher(get, acquireSearcher("get", SearcherScope.EXTERNAL, searcherWrapper)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java new file mode 100644 index 0000000000000..92dfbb826e088 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java @@ -0,0 +1,278 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.engine; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafMetaData; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Terms; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A {@link DirectoryReader} contains a single leaf reader delegating to an in-memory Lucene segment that is lazily created from + * a single document. + */ +final class SingleDocDirectoryReader extends DirectoryReader { + private final SingleDocLeafReader leafReader; + + SingleDocDirectoryReader(ShardId shardId, Translog.Index operation, DocumentMapper mapper, Analyzer analyzer) throws IOException { + this(new SingleDocLeafReader(shardId, operation, mapper, analyzer)); + } + + private SingleDocDirectoryReader(SingleDocLeafReader leafReader) throws IOException { + super(leafReader.directory, new LeafReader[]{leafReader}); + this.leafReader = leafReader; + } + + boolean assertMemorySegmentStatus(boolean loaded) { + return leafReader.assertMemorySegmentStatus(loaded); + } + + private static UnsupportedOperationException unsupported() { + assert false : "unsupported operation"; + return new UnsupportedOperationException(); + } + + @Override + protected DirectoryReader doOpenIfChanged() { + throw unsupported(); + } + + @Override + protected DirectoryReader doOpenIfChanged(IndexCommit commit) { + throw unsupported(); + } + + @Override + protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) { + throw unsupported(); + } + + @Override + public long getVersion() { + throw unsupported(); + } + + @Override + public boolean isCurrent() { + throw unsupported(); + } + + @Override + public IndexCommit getIndexCommit() { + throw unsupported(); + } + + @Override + protected void doClose() throws IOException { + leafReader.close(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return leafReader.getReaderCacheHelper(); + } + + private static class SingleDocLeafReader extends LeafReader { + + private final ShardId shardId; + private final Translog.Index operation; + private final DocumentMapper mapper; + private final Analyzer analyzer; + private final Directory directory; + private final AtomicReference delegate = new AtomicReference<>(); + + SingleDocLeafReader(ShardId shardId, Translog.Index operation, DocumentMapper mapper, Analyzer analyzer) { + this.shardId = shardId; + this.operation = operation; + this.mapper = mapper; + this.analyzer = analyzer; + this.directory = new ByteBuffersDirectory(); + } + + private LeafReader getDelegate() { + ensureOpen(); + LeafReader reader = delegate.get(); + if (reader == null) { + synchronized (this) { + reader = delegate.get(); + if (reader == null) { + reader = createInMemoryLeafReader(); + final LeafReader existing = delegate.getAndSet(reader); + assert existing == null; + } + } + } + return reader; + } + + private LeafReader createInMemoryLeafReader() { + assert Thread.holdsLock(this); + final ParsedDocument parsedDocs = mapper.parse(new SourceToParse(shardId.getIndexName(), operation.id(), + operation.source(), XContentHelper.xContentType(operation.source()), operation.routing())); + parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm()); + parsedDocs.version().setLongValue(operation.version()); + final IndexWriterConfig writeConfig = new IndexWriterConfig(analyzer).setOpenMode(IndexWriterConfig.OpenMode.CREATE); + try (IndexWriter writer = new IndexWriter(directory, writeConfig)) { + writer.addDocument(parsedDocs.rootDoc()); + final DirectoryReader reader = open(writer); + if (reader.leaves().size() != 1 || reader.leaves().get(0).reader().numDocs() != 1) { + reader.close(); + throw new IllegalStateException("Expected a single document segment; " + + "but [" + reader.leaves().size() + " segments with " + reader.leaves().get(0).reader().numDocs() + " documents"); + } + return reader.leaves().get(0).reader(); + } catch (IOException e) { + throw new EngineException(shardId, "failed to create an in-memory segment for get [" + operation.id() + "]", e); + } + } + + @Override + public CacheHelper getCoreCacheHelper() { + return getDelegate().getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return getDelegate().getReaderCacheHelper(); + } + + @Override + public Terms terms(String field) throws IOException { + return getDelegate().terms(field); + } + + @Override + public NumericDocValues getNumericDocValues(String field) throws IOException { + return getDelegate().getNumericDocValues(field); + } + + @Override + public BinaryDocValues getBinaryDocValues(String field) throws IOException { + return getDelegate().getBinaryDocValues(field); + } + + @Override + public SortedDocValues getSortedDocValues(String field) throws IOException { + return getDelegate().getSortedDocValues(field); + } + + @Override + public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException { + return getDelegate().getSortedNumericDocValues(field); + } + + @Override + public SortedSetDocValues getSortedSetDocValues(String field) throws IOException { + return getDelegate().getSortedSetDocValues(field); + } + + @Override + public NumericDocValues getNormValues(String field) throws IOException { + return getDelegate().getNormValues(field); + } + + @Override + public FieldInfos getFieldInfos() { + return getDelegate().getFieldInfos(); + } + + @Override + public Bits getLiveDocs() { + return getDelegate().getLiveDocs(); + } + + @Override + public PointValues getPointValues(String field) throws IOException { + return getDelegate().getPointValues(field); + } + + @Override + public void checkIntegrity() throws IOException { + } + + @Override + public LeafMetaData getMetaData() { + return getDelegate().getMetaData(); + } + + @Override + public Fields getTermVectors(int docID) throws IOException { + return getDelegate().getTermVectors(docID); + } + + @Override + public int numDocs() { + return 1; + } + + @Override + public int maxDoc() { + return 1; + } + + synchronized boolean assertMemorySegmentStatus(boolean loaded) { + if (loaded) { + assert delegate.get() != null : + "Expected an in memory segment was loaded; but it wasn't. Please check the reader wrapper implementation"; + } else { + assert delegate.get() == null : + "Expected an in memory segment wasn't loaded; but it was. Please check the reader wrapper implementation"; + } + return true; + } + + @Override + public void document(int docID, StoredFieldVisitor visitor) throws IOException { + assert assertMemorySegmentStatus(true); + getDelegate().document(docID, visitor); + } + + @Override + protected void doClose() throws IOException { + IOUtils.close(delegate.get(), directory); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index d2e469012cca3..498c15f243868 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -923,7 +923,7 @@ public Engine.GetResult get(Engine.Get get) { if (mapper == null) { return GetResult.NOT_EXISTS; } - return getEngine().get(get, this::acquireSearcher); + return getEngine().get(get, mapper, this::wrapSearcher); } /** diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java index 9ceda83157e10..b516e903b0ffb 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java @@ -50,7 +50,7 @@ public class PluginInfo implements Writeable, ToXContentObject { public static final String ES_PLUGIN_PROPERTIES = "plugin-descriptor.properties"; public static final String ES_PLUGIN_POLICY = "plugin-security.policy"; - private static final Version QUOTA_FS_PLUGIN_SUPPORT = Version.CURRENT; + private static final Version QUOTA_FS_PLUGIN_SUPPORT = Version.V_7_11_0; private final String name; private final String description; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AdaptingAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/AdaptingAggregator.java new file mode 100644 index 0000000000000..832ecce4ed7eb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AdaptingAggregator.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.search.profile.aggregation.InternalAggregationProfileTree; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * An {@linkplain Aggregator} that delegates collection to another + * {@linkplain Aggregator} and then translates its results into the results + * you'd expect from another aggregation. + */ +public abstract class AdaptingAggregator extends Aggregator { + private final Aggregator parent; + private final Aggregator delegate; + + public AdaptingAggregator( + Aggregator parent, + AggregatorFactories subAggregators, + CheckedFunction delegate + ) throws IOException { + // Its important we set parent first or else when we build the sub-aggregators they can fail because they'll call this.parent. + this.parent = parent; + /* + * Lock the parent of the sub-aggregators to *this* instead of to + * the delegate. This keeps the parent link shaped like the requested + * agg tree. Thisis how it has always been and some aggs rely on it. + */ + this.delegate = delegate.apply(subAggregators.fixParent(this)); + assert this.delegate.parent() == parent : "invalid parent set on delegate"; + } + + /** + * Adapt the result from the collecting {@linkplain Aggregator} into the + * result expected by this {@linkplain Aggregator}. + */ + protected abstract InternalAggregation adapt(InternalAggregation delegateResult); + + @Override + public final void close() { + delegate.close(); + } + + @Override + public final ScoreMode scoreMode() { + return delegate.scoreMode(); + } + + @Override + public final String name() { + return delegate.name(); + } + + @Override + public final Aggregator parent() { + return parent; + } + + @Override + public final Aggregator subAggregator(String name) { + return delegate.subAggregator(name); + } + + @Override + public final LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException { + return delegate.getLeafCollector(ctx); + } + + @Override + public final void preCollection() throws IOException { + delegate.preCollection(); + } + + @Override + public final InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + InternalAggregation[] delegateResults = delegate.buildAggregations(owningBucketOrds); + InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length]; + for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { + result[ordIdx] = adapt(delegateResults[ordIdx]); + } + return result; + } + + @Override + public final InternalAggregation buildEmptyAggregation() { + return adapt(delegate.buildEmptyAggregation()); + } + + @Override + public final Aggregator[] subAggregators() { + return delegate.subAggregators(); + } + + @Override + public void collectDebugInfo(BiConsumer add) { + super.collectDebugInfo(add); + add.accept("delegate", InternalAggregationProfileTree.typeFromAggregator(delegate)); + Map delegateDebug = new HashMap<>(); + delegate.collectDebugInfo(delegateDebug::put); + add.accept("delegate_debug", delegateDebug); + } + + public Aggregator delegate() { + return delegate; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java index 5e232829aaa7e..5c58b3ac0414b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java @@ -172,6 +172,11 @@ public final InternalAggregation buildTopLevel() throws IOException { */ public void collectDebugInfo(BiConsumer add) {} + /** + * Get the aggregators running under this one. + */ + public abstract Aggregator[] subAggregators(); + /** Aggregation mode for sub aggregations. */ public enum SubAggCollectionMode implements Writeable { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java index be9c81b5759b9..5cbb3225c17a4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java @@ -224,6 +224,7 @@ public Aggregator parent() { return parent; } + @Override public Aggregator[] subAggregators() { return subAggregators; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java index ffba5ca28fea5..9f9d090515a8a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java @@ -227,6 +227,27 @@ public int countAggregators() { return factories.length; } + /** + * This returns a copy of {@link AggregatorFactories} modified so that + * calls to {@link #createSubAggregators} will ignore the provided parent + * aggregator and always use {@code fixedParent} provided in to this + * method. + *

+ * {@link AdaptingAggregator} uses this to make sure that sub-aggregators + * get the {@link AdaptingAggregator} aggregator itself as the parent. + */ + public AggregatorFactories fixParent(Aggregator fixedParent) { + AggregatorFactories previous = this; + return new AggregatorFactories(factories) { + @Override + public Aggregator[] createSubAggregators(SearchContext searchContext, Aggregator parent, CardinalityUpperBound cardinality) + throws IOException { + // Note that we're throwing out the "parent" passed in to this method and using the parent passed to fixParent + return previous.createSubAggregators(searchContext, fixedParent, cardinality); + } + }; + } + /** * A mutable collection of {@link AggregationBuilder}s and * {@link PipelineAggregationBuilder}s. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java index e75b7f0c308f0..3d0bb2bec1e23 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java @@ -119,6 +119,11 @@ public Aggregator resolveSortPath(PathElement next, Iterator path) public BucketComparator bucketComparator(String key, SortOrder order) { throw new UnsupportedOperationException("Can't sort on deferred aggregations"); } + + @Override + public Aggregator[] subAggregators() { + return in.subAggregators(); + } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java index 4687ccd323d0b..ff710b99f6f23 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java @@ -20,6 +20,17 @@ package org.elasticsearch.search.aggregations.bucket.filter; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.elasticsearch.common.ParseField; @@ -45,9 +56,16 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Supplier; +import java.util.function.BiConsumer; -public class FiltersAggregator extends BucketsAggregator { +/** + * Aggregator for {@code filters}. There are two known subclasses, + * {@link FilterByFilter} which is fast but only works in some cases and + * {@link Compatible} which works in all cases. + * {@link FiltersAggregator#build} will build the fastest version that + * works with the configuration. + */ +public abstract class FiltersAggregator extends BucketsAggregator { public static final ParseField FILTERS_FIELD = new ParseField("filters"); public static final ParseField OTHER_BUCKET_FIELD = new ParseField("other_bucket"); @@ -115,58 +133,110 @@ public boolean equals(Object obj) { } } + /** + * Build an {@link Aggregator} for a {@code filters} aggregation. If there + * isn't a parent, there aren't children, and we don't collect "other" + * buckets then this will a faster {@link FilterByFilter} aggregator. + * Otherwise it'll fall back to a slower aggregator that is + * {@link Compatible} with parent, children, and "other" buckets. + */ + public static FiltersAggregator build( + String name, + AggregatorFactories factories, + String[] keys, + Query[] filters, + boolean keyed, + String otherBucketKey, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + FiltersAggregator filterOrder = buildFilterOrderOrNull( + name, + factories, + keys, + filters, + keyed, + otherBucketKey, + context, + parent, + cardinality, + metadata + ); + if (filterOrder != null) { + return filterOrder; + } + return new FiltersAggregator.Compatible( + name, + factories, + keys, + filters, + keyed, + otherBucketKey, + context, + parent, + cardinality, + metadata + ); + } + + /** + * Build an {@link Aggregator} for a {@code filters} aggregation if we + * can collect {@link FilterByFilter}, otherwise return {@code null}. We can + * collect filter by filter if there isn't a parent, there aren't children, + * and we don't collect "other" buckets. Collecting {@link FilterByFilter} + * is generally going to be much faster than the {@link Compatible} aggregator. + */ + public static FiltersAggregator buildFilterOrderOrNull( + String name, + AggregatorFactories factories, + String[] keys, + Query[] filters, + boolean keyed, + String otherBucketKey, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + if (parent != null) { + return null; + } + if (factories.countAggregators() != 0) { + return null; + } + if (otherBucketKey != null) { + return null; + } + return new FiltersAggregator.FilterByFilter( + name, + keys, + filters, + keyed, + context, + parent, + cardinality, + metadata + ); + } + private final String[] keys; - private Supplier filters; private final boolean keyed; - private final boolean showOtherBucket; - private final String otherBucketKey; - private final int totalNumKeys; + protected final String otherBucketKey; - public FiltersAggregator(String name, AggregatorFactories factories, String[] keys, Supplier filters, boolean keyed, + private FiltersAggregator(String name, AggregatorFactories factories, String[] keys, boolean keyed, String otherBucketKey, SearchContext context, Aggregator parent, CardinalityUpperBound cardinality, Map metadata) throws IOException { super(name, factories, context, parent, cardinality.multiply(keys.length + (otherBucketKey == null ? 0 : 1)), metadata); this.keyed = keyed; this.keys = keys; - this.filters = filters; - this.showOtherBucket = otherBucketKey != null; this.otherBucketKey = otherBucketKey; - if (showOtherBucket) { - this.totalNumKeys = keys.length + 1; - } else { - this.totalNumKeys = keys.length; - } - } - - @Override - public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, - final LeafBucketCollector sub) throws IOException { - // no need to provide deleted docs to the filter - Weight[] filters = this.filters.get(); - final Bits[] bits = new Bits[filters.length]; - for (int i = 0; i < filters.length; ++i) { - bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filters[i].scorerSupplier(ctx)); - } - return new LeafBucketCollectorBase(sub, null) { - @Override - public void collect(int doc, long bucket) throws IOException { - boolean matched = false; - for (int i = 0; i < bits.length; i++) { - if (bits[i].get(doc)) { - collectBucket(sub, doc, bucketOrd(bucket, i)); - matched = true; - } - } - if (showOtherBucket && !matched) { - collectBucket(sub, doc, bucketOrd(bucket, bits.length)); - } - } - }; } @Override public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { - return buildAggregationsForFixedBucketCount(owningBucketOrds, keys.length + (showOtherBucket ? 1 : 0), + return buildAggregationsForFixedBucketCount(owningBucketOrds, keys.length + (otherBucketKey == null ? 0 : 1), (offsetInOwningOrd, docCount, subAggregationResults) -> { if (offsetInOwningOrd < keys.length) { return new InternalFilters.InternalBucket(keys[offsetInOwningOrd], docCount, @@ -185,7 +255,7 @@ public InternalAggregation buildEmptyAggregation() { buckets.add(bucket); } - if (showOtherBucket) { + if (otherBucketKey != null) { InternalFilters.InternalBucket bucket = new InternalFilters.InternalBucket(otherBucketKey, 0, subAggs, keyed); buckets.add(bucket); } @@ -193,8 +263,174 @@ public InternalAggregation buildEmptyAggregation() { return new InternalFilters(name, buckets, keyed, metadata()); } - final long bucketOrd(long owningBucketOrdinal, int filterOrd) { - return owningBucketOrdinal * totalNumKeys + filterOrd; + /** + * Collects results by running each filter against the searcher and doesn't + * build any {@link LeafBucketCollector}s which is generally faster than + * {@link Compatible} but doesn't support when there is a parent aggregator + * or any child aggregators. + */ + private static class FilterByFilter extends FiltersAggregator { + private final Query[] filters; + private Weight[] filterWeights; + private int segmentsWithDeletedDocs; + + FilterByFilter( + String name, + String[] keys, + Query[] filters, + boolean keyed, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super(name, AggregatorFactories.EMPTY, keys, keyed, null, context, parent, cardinality, metadata); + this.filters = filters; + } + + /** + * Instead of returning a {@link LeafBucketCollector} we do the + * collection ourselves by running the filters directly. This is safe + * because we only use this aggregator if there isn't a {@code parent} + * which would change how we collect buckets and because we take the + * top level query into account when building the filters. + */ + @Override + protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + if (filterWeights == null) { + filterWeights = buildWeights(topLevelQuery(), filters); + } + Bits live = ctx.reader().getLiveDocs(); + for (int filterOrd = 0; filterOrd < filters.length; filterOrd++) { + BulkScorer scorer = filterWeights[filterOrd].bulkScorer(ctx); + if (scorer == null) { + // the filter doesn't match any docs + continue; + } + TotalHitCountCollector collector = new TotalHitCountCollector(); + scorer.score(collector, live); + incrementBucketDocCount(filterOrd, collector.getTotalHits()); + } + // Throwing this exception is how we communicate to the collection mechanism that we don't need the segment. + throw new CollectionTerminatedException(); + } + + @Override + public void collectDebugInfo(BiConsumer add) { + super.collectDebugInfo(add); + add.accept("segments_with_deleted_docs", segmentsWithDeletedDocs); + } + } + + /** + * Collects results by building a {@link Bits} per filter and testing if + * each doc sent to its {@link LeafBucketCollector} is in each filter + * which is generally slower than {@link FilterByFilter} but is compatible + * with parent and child aggregations. + */ + private static class Compatible extends FiltersAggregator { + private final Query[] filters; + private Weight[] filterWeights; + + private final int totalNumKeys; + + Compatible( + String name, + AggregatorFactories factories, + String[] keys, + Query[] filters, + boolean keyed, + String otherBucketKey, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super(name, factories, keys, keyed, otherBucketKey, context, parent, cardinality, metadata); + this.filters = filters; + if (otherBucketKey == null) { + this.totalNumKeys = keys.length; + } else { + this.totalNumKeys = keys.length + 1; + } + } + + @Override + protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + if (filterWeights == null) { + filterWeights = buildWeights(new MatchAllDocsQuery(), filters); + } + final Bits[] bits = new Bits[filters.length]; + for (int i = 0; i < filters.length; ++i) { + bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filterWeights[i].scorerSupplier(ctx)); + } + return new LeafBucketCollectorBase(sub, null) { + @Override + public void collect(int doc, long bucket) throws IOException { + boolean matched = false; + for (int i = 0; i < bits.length; i++) { + if (bits[i].get(doc)) { + collectBucket(sub, doc, bucketOrd(bucket, i)); + matched = true; + } + } + if (otherBucketKey != null && false == matched) { + collectBucket(sub, doc, bucketOrd(bucket, bits.length)); + } + } + }; + } + + final long bucketOrd(long owningBucketOrdinal, int filterOrd) { + return owningBucketOrdinal * totalNumKeys + filterOrd; + } + } + + protected Weight[] buildWeights(Query topLevelQuery, Query filters[]) throws IOException{ + Weight[] weights = new Weight[filters.length]; + for (int i = 0; i < filters.length; ++i) { + Query filter = filterMatchingBoth(topLevelQuery, filters[i]); + weights[i] = searcher().createWeight(searcher().rewrite(filter), ScoreMode.COMPLETE_NO_SCORES, 1); + } + return weights; } + /** + * Make a filter that matches both queries, merging the + * {@link PointRangeQuery}s together if possible. The "merging together" + * part is provides a fairly substantial speed boost then executing a + * top level query on a date and a filter on a date. This kind of thing + * is very common when visualizing logs and metrics. + */ + private Query filterMatchingBoth(Query lhs, Query rhs) { + if (lhs instanceof MatchAllDocsQuery) { + return rhs; + } + if (rhs instanceof MatchAllDocsQuery) { + return lhs; + } + Query unwrappedLhs = unwrap(lhs); + Query unwrappedRhs = unwrap(rhs); + if (unwrappedLhs instanceof PointRangeQuery && unwrappedRhs instanceof PointRangeQuery) { + Query merged = MergedPointRangeQuery.merge((PointRangeQuery) unwrappedLhs, (PointRangeQuery) unwrappedRhs); + if (merged != null) { + // Should we rewrap here? + return merged; + } + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(lhs, BooleanClause.Occur.MUST); + builder.add(rhs, BooleanClause.Occur.MUST); + return builder.build(); + } + + private Query unwrap(Query query) { + if (query instanceof IndexSortSortedNumericDocValuesRangeQuery) { + query = ((IndexSortSortedNumericDocValuesRangeQuery) query).getFallbackQuery(); + } + if (query instanceof IndexOrDocValuesQuery) { + query = ((IndexOrDocValuesQuery) query).getIndexQuery(); + } + return query; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java index 83d49dd12d626..0594286bd77f3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java @@ -19,11 +19,8 @@ package org.elasticsearch.search.aggregations.bucket.filter; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; -import org.elasticsearch.search.aggregations.AggregationInitializationException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -66,31 +63,18 @@ public FiltersAggregatorFactory(String name, List filters, boolean * necessary. This is done lazily so that the {@link Weight}s are only * created if the aggregation collects documents reducing the overhead of * the aggregation in the case where no documents are collected. - * - * Note that as aggregations are initialsed and executed in a serial manner, + *

+ * Note that as aggregations are initialized and executed in a serial manner, * no concurrency considerations are necessary here. */ - public Weight[] getWeights(SearchContext searchContext) { - if (weights == null) { - try { - IndexSearcher contextSearcher = searchContext.searcher(); - weights = new Weight[filters.length]; - for (int i = 0; i < filters.length; ++i) { - this.weights[i] = contextSearcher.createWeight(contextSearcher.rewrite(filters[i]), ScoreMode.COMPLETE_NO_SCORES, 1); - } - } catch (IOException e) { - throw new AggregationInitializationException("Failed to initialse filters for aggregation [" + name() + "]", e); - } - } - return weights; - } + @Override public Aggregator createInternal(SearchContext searchContext, Aggregator parent, CardinalityUpperBound cardinality, Map metadata) throws IOException { - return new FiltersAggregator(name, factories, keys, () -> getWeights(searchContext), keyed, + return FiltersAggregator.build(name, factories, keys, filters, keyed, otherBucket ? otherBucketKey : null, searchContext, parent, cardinality, metadata); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java new file mode 100644 index 0000000000000..b21cf396ede81 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java @@ -0,0 +1,209 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.filter; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.Weight; + +import java.io.IOException; +import java.util.Objects; + +import static java.util.Arrays.compareUnsigned; + +/** + * Query merging two point in range queries. + */ +public class MergedPointRangeQuery extends Query { + /** + * Merge two {@linkplain PointRangeQuery}s into a {@linkplain MergedPointRangeQuery} + * that matches points that match both filters. + */ + public static Query merge(PointRangeQuery lhs, PointRangeQuery rhs) { + if (lhs.equals(rhs)) { + // Lucky case! The queries were the same so their UNION is just the query itself. + return lhs; + } + if (lhs.getField() != rhs.getField() || lhs.getNumDims() != rhs.getNumDims() || lhs.getBytesPerDim() != rhs.getBytesPerDim()) { + return null; + } + return new MergedPointRangeQuery(lhs, rhs); + } + + private final String field; + private final Query delegateForMultiValuedSegments; + private final Query delegateForSingleValuedSegments; + + private MergedPointRangeQuery(PointRangeQuery lhs, PointRangeQuery rhs) { + field = lhs.getField(); + delegateForMultiValuedSegments = new BooleanQuery.Builder().add(lhs, Occur.MUST).add(rhs, Occur.MUST).build(); + int numDims = lhs.getNumDims(); + int bytesPerDim = lhs.getBytesPerDim(); + this.delegateForSingleValuedSegments = pickDelegateForSingleValuedSegments( + mergeBound(lhs.getLowerPoint(), rhs.getLowerPoint(), numDims, bytesPerDim, true), + mergeBound(lhs.getUpperPoint(), rhs.getUpperPoint(), numDims, bytesPerDim, false), + numDims, + bytesPerDim + ); + } + + private Query pickDelegateForSingleValuedSegments(byte[] lower, byte[] upper, int numDims, int bytesPerDim) { + // If we ended up with disjoint ranges in any dimension then on single valued segments we can't match any docs. + for (int dim = 0; dim < numDims; dim++) { + int offset = dim * bytesPerDim; + if (compareUnsigned(lower, offset, offset + bytesPerDim, upper, offset, offset + bytesPerDim) > 0) { + return new MatchNoDocsQuery("disjoint ranges"); + } + } + // Otherwise on single valued segments we can only match docs the match the UNION of the two ranges. + return new PointRangeQuery(field, lower, upper, numDims) { + @Override + protected String toString(int dimension, byte[] value) { + // Stolen from Lucene's Binary range query. It'd be best to delegate, but the method isn't visible. + StringBuilder sb = new StringBuilder(); + sb.append("("); + for (int i = 0; i < value.length; i++) { + if (i > 0) { + sb.append(' '); + } + sb.append(Integer.toHexString(value[i] & 0xFF)); + } + sb.append(')'); + return sb.toString(); + } + }; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + Weight multiValuedSegmentWeight; + Weight singleValuedSegmentWeight; + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + ScorerSupplier scorerSupplier = scorerSupplier(context); + if (scorerSupplier == null) { + return null; + } + return scorerSupplier.get(Long.MAX_VALUE); + } + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + /* + * If we're sure docs only have a single value for the field + * we can pick the most compact bounds. If there are multiple values + * for the field we have to run the boolean query. + */ + PointValues points = context.reader().getPointValues(field); + if (points == null) { + return null; + } + if (points.size() == points.getDocCount()) { + // Each doc that has points has exactly one point. + return singleValuedSegmentWeight().scorerSupplier(context); + } + return multiValuedSegmentWeight().scorerSupplier(context); + } + + @Override + public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { + PointValues points = context.reader().getPointValues(field); + if (points == null) { + return null; + } + if (points.size() == points.getDocCount()) { + // Each doc that has points has exactly one point. + return singleValuedSegmentWeight().bulkScorer(context); + } + return multiValuedSegmentWeight().bulkScorer(context); + } + + private Weight singleValuedSegmentWeight() throws IOException { + if (singleValuedSegmentWeight == null) { + singleValuedSegmentWeight = delegateForSingleValuedSegments.createWeight(searcher, scoreMode, boost); + } + return singleValuedSegmentWeight; + } + + private Weight multiValuedSegmentWeight() throws IOException { + if (multiValuedSegmentWeight == null) { + multiValuedSegmentWeight = delegateForMultiValuedSegments.createWeight(searcher, scoreMode, boost); + } + return multiValuedSegmentWeight; + } + }; + } + + /** + * The query used when we have single valued segments. + */ + Query delegateForSingleValuedSegments() { + return delegateForSingleValuedSegments; + } + + @Override + public String toString(String field) { + return "MergedPointRange[" + delegateForMultiValuedSegments.toString(field) + "]"; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + MergedPointRangeQuery other = (MergedPointRangeQuery) obj; + return delegateForMultiValuedSegments.equals(other.delegateForMultiValuedSegments) + && delegateForSingleValuedSegments.equals(other.delegateForSingleValuedSegments); + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), delegateForMultiValuedSegments, delegateForSingleValuedSegments); + } + + private static byte[] mergeBound(byte[] lhs, byte[] rhs, int numDims, int bytesPerDim, boolean lower) { + byte[] merged = new byte[lhs.length]; + for (int dim = 0; dim < numDims; dim++) { + int offset = dim * bytesPerDim; + boolean cmp = compareUnsigned(lhs, offset, offset + bytesPerDim, rhs, offset, offset + bytesPerDim) <= 0; + byte[] from = (cmp ^ lower) ? lhs : rhs; + System.arraycopy(from, offset, merged, offset, bytesPerDim); + } + return merged; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java index 0ce24385f72b7..125ca3d2c50c6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java @@ -22,35 +22,184 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.Rounding.DateTimeUnit; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AdaptingAggregator; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; +import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator; +import org.elasticsearch.search.aggregations.bucket.range.InternalDateRange; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator; +import org.elasticsearch.search.aggregations.bucket.range.RangeAggregatorSupplier; import org.elasticsearch.search.aggregations.bucket.terms.LongKeyedBucketOrds; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.function.BiConsumer; /** - * An aggregator for date values. Every date is rounded down using a configured - * {@link Rounding}. - * - * @see Rounding + * Aggregator for {@code date_histogram} that rounds values using + * {@link Rounding}. See {@link FromDateRange} which also aggregates for + * {@code date_histogram} but does so by running a {@code range} aggregation + * over the date and transforming the results. In general + * {@link FromDateRange} is faster than {@link DateHistogramAggregator} + * but {@linkplain DateHistogramAggregator} works when we can't precalculate + * all of the {@link Rounding.Prepared#fixedRoundingPoints() fixed rounding points}. */ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAggregator { + /** + * Build an {@link Aggregator} for a {@code date_histogram} aggregation. + * If we can determine the bucket boundaries from + * {@link Rounding.Prepared#fixedRoundingPoints()} we use + * {@link RangeAggregator} to do the actual collecting, otherwise we use + * an specialized {@link DateHistogramAggregator Aggregator} specifically + * for the {@code date_histogram}s. We prefer to delegate to the + * {@linkplain RangeAggregator} because it can sometimes be further + * optimized into a {@link FiltersAggregator}. Even when it can't be + * optimized, it is going to be marginally faster and consume less memory + * than the {@linkplain DateHistogramAggregator} because it doesn't need + * to the round points and because it can pass precise cardinality + * estimates to its child aggregations. + */ + public static Aggregator build( + String name, + AggregatorFactories factories, + Rounding rounding, + BucketOrder order, + boolean keyed, + long minDocCount, + @Nullable LongBounds extendedBounds, + @Nullable LongBounds hardBounds, + ValuesSourceConfig valuesSourceConfig, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + Rounding.Prepared preparedRounding = valuesSourceConfig.roundingPreparer().apply(rounding); + Aggregator asRange = adaptIntoRangeOrNull( + name, + factories, + rounding, + preparedRounding, + order, + keyed, + minDocCount, + extendedBounds, + hardBounds, + valuesSourceConfig, + context, + parent, + cardinality, + metadata + ); + if (asRange != null) { + return asRange; + } + return new DateHistogramAggregator( + name, + factories, + rounding, + preparedRounding, + order, + keyed, + minDocCount, + extendedBounds, + hardBounds, + valuesSourceConfig, + context, + parent, + cardinality, + metadata + ); + } + + private static FromDateRange adaptIntoRangeOrNull( + String name, + AggregatorFactories factories, + Rounding rounding, + Rounding.Prepared preparedRounding, + BucketOrder order, + boolean keyed, + long minDocCount, + @Nullable LongBounds extendedBounds, + @Nullable LongBounds hardBounds, + ValuesSourceConfig valuesSourceConfig, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + if (hardBounds != null || extendedBounds != null) { + return null; + } + long[] fixedRoundingPoints = preparedRounding.fixedRoundingPoints(); + if (fixedRoundingPoints == null) { + return null; + } + // Range aggs use a double to aggregate and we don't want to lose precision. + long min = fixedRoundingPoints[0]; + long max = fixedRoundingPoints[fixedRoundingPoints.length - 1]; + if (min < -RangeAggregator.MAX_ACCURATE_BOUND || min > RangeAggregator.MAX_ACCURATE_BOUND) { + return null; + } + if (max < -RangeAggregator.MAX_ACCURATE_BOUND || max > RangeAggregator.MAX_ACCURATE_BOUND) { + return null; + } + RangeAggregatorSupplier rangeSupplier = context.getQueryShardContext() + .getValuesSourceRegistry() + .getAggregator(RangeAggregationBuilder.REGISTRY_KEY, valuesSourceConfig); + if (rangeSupplier == null) { + return null; + } + RangeAggregator.Range[] ranges = new RangeAggregator.Range[fixedRoundingPoints.length]; + for (int i = 0; i < fixedRoundingPoints.length - 1; i++) { + ranges[i] = new RangeAggregator.Range(null, (double) fixedRoundingPoints[i], (double) fixedRoundingPoints[i + 1]); + } + ranges[ranges.length - 1] = new RangeAggregator.Range(null, (double) fixedRoundingPoints[fixedRoundingPoints.length - 1], null); + return new DateHistogramAggregator.FromDateRange( + parent, + factories, + subAggregators -> rangeSupplier.build( + name, + subAggregators, + valuesSourceConfig, + InternalDateRange.FACTORY, + ranges, + false, + context, + parent, + cardinality, + metadata + ), + valuesSourceConfig.format(), + rounding, + preparedRounding, + order, + minDocCount, + extendedBounds, + keyed, + fixedRoundingPoints + ); + } private final ValuesSource.Numeric valuesSource; private final DocValueFormat formatter; @@ -72,6 +221,7 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg String name, AggregatorFactories factories, Rounding rounding, + Rounding.Prepared preparedRounding, BucketOrder order, boolean keyed, long minDocCount, @@ -86,7 +236,7 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg super(name, factories, aggregationContext, parent, CardinalityUpperBound.MANY, metadata); this.rounding = rounding; - this.preparedRounding = valuesSourceConfig.roundingPreparer().apply(rounding); + this.preparedRounding = preparedRounding; this.order = order; order.validate(this); this.keyed = keyed; @@ -153,8 +303,6 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator()); - // value source will be null for unmapped fields - // Important: use `rounding` here, not `shardRounding` InternalDateHistogram.EmptyBucketInfo emptyBucketInfo = minDocCount == 0 ? new InternalDateHistogram.EmptyBucketInfo(rounding.withoutOffset(), buildEmptySubAggregations(), extendedBounds) : null; @@ -195,4 +343,93 @@ public double bucketSize(long bucket, Rounding.DateTimeUnit unitSize) { return 1.0; } } + + static class FromDateRange extends AdaptingAggregator implements SizedBucketAggregator { + private final DocValueFormat format; + private final Rounding rounding; + private final Rounding.Prepared preparedRounding; + private final BucketOrder order; + private final long minDocCount; + private final LongBounds extendedBounds; + private final boolean keyed; + private final long[] fixedRoundingPoints; + + FromDateRange( + Aggregator parent, + AggregatorFactories subAggregators, + CheckedFunction delegate, + DocValueFormat format, + Rounding rounding, + Rounding.Prepared preparedRounding, + BucketOrder order, + long minDocCount, + LongBounds extendedBounds, + boolean keyed, + long[] fixedRoundingPoints + ) throws IOException { + super(parent, subAggregators, delegate); + this.format = format; + this.rounding = rounding; + this.preparedRounding = preparedRounding; + this.order = order; + order.validate(this); + this.minDocCount = minDocCount; + this.extendedBounds = extendedBounds; + this.keyed = keyed; + this.fixedRoundingPoints = fixedRoundingPoints; + } + + @Override + protected InternalAggregation adapt(InternalAggregation delegateResult) { + InternalDateRange range = (InternalDateRange) delegateResult; + List buckets = new ArrayList<>(range.getBuckets().size()); + for (InternalDateRange.Bucket rangeBucket : range.getBuckets()) { + if (rangeBucket.getDocCount() > 0) { + buckets.add( + new InternalDateHistogram.Bucket( + rangeBucket.getFrom().toInstant().toEpochMilli(), + rangeBucket.getDocCount(), + keyed, + format, + rangeBucket.getAggregations() + ) + ); + } + } + CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator()); + + InternalDateHistogram.EmptyBucketInfo emptyBucketInfo = minDocCount == 0 + ? new InternalDateHistogram.EmptyBucketInfo(rounding.withoutOffset(), buildEmptySubAggregations(), extendedBounds) + : null; + return new InternalDateHistogram( + range.getName(), + buckets, + order, + minDocCount, + rounding.offset(), + emptyBucketInfo, + format, + keyed, + range.getMetadata() + ); + } + + public final InternalAggregations buildEmptySubAggregations() { + List aggs = new ArrayList<>(); + for (Aggregator aggregator : subAggregators()) { + aggs.add(aggregator.buildEmptyAggregation()); + } + return InternalAggregations.from(aggs); + } + + @Override + public double bucketSize(long bucket, DateTimeUnit unitSize) { + if (unitSize != null) { + long startPoint = bucket < fixedRoundingPoints.length ? fixedRoundingPoints[(int) bucket] : Long.MIN_VALUE; + return preparedRounding.roundingSize(startPoint, unitSize); + } else { + return 1.0; + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java index a5518c8a3eb4a..d069a7cb8c7a7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java @@ -42,7 +42,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( DateHistogramAggregationBuilder.REGISTRY_KEY, List.of(CoreValuesSourceType.DATE, CoreValuesSourceType.NUMERIC, CoreValuesSourceType.BOOLEAN), - DateHistogramAggregator::new, + DateHistogramAggregator::build, true); builder.register(DateHistogramAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.RANGE, DateRangeHistogramAggregator::new, true); @@ -111,7 +111,7 @@ protected Aggregator doCreateInternal( protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { - return new DateHistogramAggregator(name, factories, rounding, order, keyed, minDocCount, extendedBounds, hardBounds, + return new DateHistogramAggregator(name, factories, rounding, null, order, keyed, minDocCount, extendedBounds, hardBounds, config, searchContext, parent, CardinalityUpperBound.NONE, metadata); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java index 6790cb5ee9147..1072ad54382ae 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Unmapped; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; @@ -51,7 +50,7 @@ public static void registerAggregators( builder.register( registryKey, List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN), - RangeAggregator::new, + RangeAggregator::build, true); } @@ -92,8 +91,7 @@ protected Aggregator doCreateInternal( .build( name, factories, - (Numeric) config.getValuesSource(), - config.format(), + config, rangeFactory, ranges, keyed, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java index 790ae59c66b31..8e37af016987e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java @@ -66,7 +66,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { cardinality, metadata) -> { DistanceSource distanceSource = new DistanceSource((ValuesSource.GeoPoint) valuesSource, distanceType, origin, units); - return new RangeAggregator( + return RangeAggregator.buildWithoutAttemptedToAdaptToFilters( name, factories, distanceSource, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java index 2c937ab104c54..fd68e3e1ac4b5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.time.Instant; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.List; import java.util.Map; @@ -46,13 +47,13 @@ public Bucket(String key, double from, double to, long docCount, InternalAggrega } @Override - public Object getFrom() { + public ZonedDateTime getFrom() { return Double.isInfinite(((Number) from).doubleValue()) ? null : Instant.ofEpochMilli(((Number) from).longValue()).atZone(ZoneOffset.UTC); } @Override - public Object getTo() { + public ZonedDateTime getTo() { return Double.isInfinite(((Number) to).doubleValue()) ? null : Instant.ofEpochMilli(((Number) to).longValue()).atZone(ZoneOffset.UTC); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java index c750fdc2d062c..1ba14d7a0715c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; @@ -113,7 +112,7 @@ public long getDocCount() { } @Override - public Aggregations getAggregations() { + public InternalAggregations getAggregations() { return aggregations; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java index 82005aaaf40d4..d28107318e788 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java @@ -19,8 +19,11 @@ package org.elasticsearch.search.aggregations.bucket.range; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -31,7 +34,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AdaptingAggregator; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.CardinalityUpperBound; @@ -41,7 +47,12 @@ import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; +import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator; +import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; +import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory; import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -52,7 +63,21 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class RangeAggregator extends BucketsAggregator { +/** + * Aggregator for {@code range}. There are two known subclasses, + * {@link NoOverlap} which is fast but only compatible with ranges that + * don't have overlaps and {@link Overlap} which handles overlapping + * ranges. There is also {@link FromFilters} which isn't a subclass + * but is also a functional aggregator for {@code range}. + * {@link RangeAggregator#build} will build the fastest of the three + * that is compatible with the requested configuration. + */ +public abstract class RangeAggregator extends BucketsAggregator { + /** + * The maximum {@code long} that can accurately fit into the + * {@code double} precision floating point bounds. + */ + public static final long MAX_ACCURATE_BOUND = 1L << 53; public static final ParseField RANGES_FIELD = new ParseField("ranges"); public static final ParseField KEYED_FIELD = new ParseField("keyed"); @@ -215,15 +240,198 @@ public boolean equals(Object obj) { } } - final ValuesSource.Numeric valuesSource; - final DocValueFormat format; - final Range[] ranges; - final boolean keyed; - final InternalRange.Factory rangeFactory; + /** + * Build an {@link Aggregator} for a {@code range} aggregation. If the + * {@code ranges} can be converted into filters then it builds a + * {@link FiltersAggregator} and uses that to collect the results + * if that aggregator can run in "filter by filter" + * collection mode. If it can't then we'll collect the ranges using + * a native {@link RangeAggregator} which is significantly faster + * than the "compatible" collection mechanism for the filters agg. + */ + public static Aggregator build( + String name, + AggregatorFactories factories, + ValuesSourceConfig valuesSourceConfig, + InternalRange.Factory rangeFactory, + Range[] ranges, + boolean keyed, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + Aggregator adapted = adaptIntoFiltersOrNull( + name, + factories, + valuesSourceConfig, + rangeFactory, + ranges, + keyed, + context, + parent, + cardinality, + metadata + ); + if (adapted != null) { + return adapted; + } + return buildWithoutAttemptedToAdaptToFilters( + name, + factories, + (ValuesSource.Numeric) valuesSourceConfig.getValuesSource(), + valuesSourceConfig.format(), + rangeFactory, + ranges, + keyed, + context, + parent, + cardinality, + metadata + ); + } + + public static Aggregator adaptIntoFiltersOrNull( + String name, + AggregatorFactories factories, + ValuesSourceConfig valuesSourceConfig, + InternalRange.Factory rangeFactory, + Range[] ranges, + boolean keyed, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + if (false == valuesSourceConfig.alignesWithSearchIndex()) { + return null; + } + // TODO bail here for runtime fields. They'll be slower this way. Maybe we can somehow look at the Query? + if (valuesSourceConfig.fieldType() instanceof DateFieldType + && ((DateFieldType) valuesSourceConfig.fieldType()).resolution() == Resolution.NANOSECONDS) { + // We don't generate sensible Queries for nanoseconds. + return null; + } + boolean wholeNumbersOnly = false == ((ValuesSource.Numeric) valuesSourceConfig.getValuesSource()).isFloatingPoint(); + String[] keys = new String[ranges.length]; + Query[] filters = new Query[ranges.length]; + for (int i = 0; i < ranges.length; i++) { + /* + * If the bounds on the ranges are too high then the `double`s + * that we work with will round differently in the native range + * aggregator than in the filters aggregator. So we can't use + * the filters. That is, if the input data type is a `long` in + * the first place. If it isn't then + */ + if (wholeNumbersOnly && ranges[i].from != Double.NEGATIVE_INFINITY && Math.abs(ranges[i].from) > MAX_ACCURATE_BOUND) { + return null; + } + if (wholeNumbersOnly && ranges[i].to != Double.POSITIVE_INFINITY && Math.abs(ranges[i].to) > MAX_ACCURATE_BOUND) { + return null; + } + keys[i] = Integer.toString(i); + /* + * Use the native format on the field rather than the one provided + * on the valuesSourceConfig because the format on the field is what + * we parse. With https://github.com/elastic/elasticsearch/pull/63692 + * we can just cast to a long here and it'll be taken as millis. + */ + DocValueFormat format = valuesSourceConfig.fieldType().docValueFormat(null, null); + // TODO correct the loss of precision from the range somehow.....? + filters[i] = valuesSourceConfig.fieldType() + .rangeQuery( + ranges[i].from == Double.NEGATIVE_INFINITY ? null : format.format(ranges[i].from), + ranges[i].to == Double.POSITIVE_INFINITY ? null : format.format(ranges[i].to), + true, + false, + ShapeRelation.CONTAINS, + null, + null, + context.getQueryShardContext() + ); + } + FiltersAggregator delegate = FiltersAggregator.buildFilterOrderOrNull( + name, + factories, + keys, + filters, + false, + null, + context, + parent, + cardinality, + metadata + ); + if (delegate == null) { + return null; + } + RangeAggregator.FromFilters fromFilters = new RangeAggregator.FromFilters<>( + parent, + factories, + subAggregators -> { + if (subAggregators.countAggregators() > 0) { + throw new IllegalStateException("didn't expect to have a delegate if there are child aggs"); + } + return delegate; + }, + valuesSourceConfig.format(), + ranges, + keyed, + rangeFactory + ); + return fromFilters; + } + + public static Aggregator buildWithoutAttemptedToAdaptToFilters( + String name, + AggregatorFactories factories, + ValuesSource.Numeric valuesSource, + DocValueFormat format, + InternalRange.Factory rangeFactory, + Range[] ranges, + boolean keyed, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + if (hasOverlap(ranges)) { + return new RangeAggregator.Overlap( + name, + factories, + valuesSource, + format, + rangeFactory, + ranges, + keyed, + context, + parent, + cardinality, + metadata + ); + } + return new RangeAggregator.NoOverlap( + name, + factories, + valuesSource, + format, + rangeFactory, + ranges, + keyed, + context, + parent, + cardinality, + metadata + ); + } - final double[] maxTo; + private final ValuesSource.Numeric valuesSource; + private final DocValueFormat format; + protected final Range[] ranges; + private final boolean keyed; + private final InternalRange.Factory rangeFactory; - public RangeAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format, + private RangeAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format, InternalRange.Factory rangeFactory, Range[] ranges, boolean keyed, SearchContext context, Aggregator parent, CardinalityUpperBound cardinality, Map metadata) throws IOException { @@ -233,15 +441,7 @@ public RangeAggregator(String name, AggregatorFactories factories, ValuesSource. this.format = format; this.keyed = keyed; this.rangeFactory = rangeFactory; - this.ranges = ranges; - - maxTo = new double[this.ranges.length]; - maxTo[0] = this.ranges[0].to; - for (int i = 1; i < this.ranges.length; ++i) { - maxTo[i] = Math.max(this.ranges[i].to,maxTo[i-1]); - } - } @Override @@ -253,8 +453,7 @@ public ScoreMode scoreMode() { } @Override - public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, - final LeafBucketCollector sub) throws IOException { + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override @@ -263,63 +462,14 @@ public void collect(int doc, long bucket) throws IOException { final int valuesCount = values.docValueCount(); for (int i = 0, lo = 0; i < valuesCount; ++i) { final double value = values.nextValue(); - lo = collect(doc, value, bucket, lo); + lo = RangeAggregator.this.collect(sub, doc, value, bucket, lo); } } } - - private int collect(int doc, double value, long owningBucketOrdinal, int lowBound) throws IOException { - int lo = lowBound, hi = ranges.length - 1; // all candidates are between these indexes - int mid = (lo + hi) >>> 1; - while (lo <= hi) { - if (value < ranges[mid].from) { - hi = mid - 1; - } else if (value >= maxTo[mid]) { - lo = mid + 1; - } else { - break; - } - mid = (lo + hi) >>> 1; - } - if (lo > hi) return lo; // no potential candidate - - // binary search the lower bound - int startLo = lo, startHi = mid; - while (startLo <= startHi) { - final int startMid = (startLo + startHi) >>> 1; - if (value >= maxTo[startMid]) { - startLo = startMid + 1; - } else { - startHi = startMid - 1; - } - } - - // binary search the upper bound - int endLo = mid, endHi = hi; - while (endLo <= endHi) { - final int endMid = (endLo + endHi) >>> 1; - if (value < ranges[endMid].from) { - endHi = endMid - 1; - } else { - endLo = endMid + 1; - } - } - - assert startLo == lowBound || value >= maxTo[startLo - 1]; - assert endHi == ranges.length - 1 || value < ranges[endHi + 1].from; - - for (int i = startLo; i <= endHi; ++i) { - if (ranges[i].matches(value)) { - collectBucket(sub, doc, subBucketOrdinal(owningBucketOrdinal, i)); - } - } - - return endHi + 1; - } }; } - private long subBucketOrdinal(long owningBucketOrdinal, int rangeOrd) { + protected long subBucketOrdinal(long owningBucketOrdinal, int rangeOrd) { return owningBucketOrdinal * ranges.length + rangeOrd; } @@ -383,4 +533,179 @@ public InternalAggregation buildEmptyAggregation() { } } + protected abstract int collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, int lowBound) + throws IOException; + + private static class NoOverlap extends RangeAggregator { + NoOverlap( + String name, + AggregatorFactories factories, + Numeric valuesSource, + DocValueFormat format, + Factory rangeFactory, + Range[] ranges, + boolean keyed, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super(name, factories, valuesSource, format, rangeFactory, ranges, keyed, context, parent, cardinality, metadata); + } + + @Override + protected int collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, int lowBound) throws IOException { + int lo = lowBound, hi = ranges.length - 1; + while (lo <= hi) { + final int mid = (lo + hi) >>> 1; + if (value < ranges[mid].from) { + hi = mid - 1; + } else if (value >= ranges[mid].to) { + lo = mid + 1; + } else { + collectBucket(sub, doc, subBucketOrdinal(owningBucketOrdinal, mid)); + // The next value must fall in the next bucket to be collected. + return mid + 1; + } + } + return lo; + } + } + + private static class Overlap extends RangeAggregator { + Overlap( + String name, + AggregatorFactories factories, + Numeric valuesSource, + DocValueFormat format, + Factory rangeFactory, + Range[] ranges, + boolean keyed, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super(name, factories, valuesSource, format, rangeFactory, ranges, keyed, context, parent, cardinality, metadata); + maxTo = new double[ranges.length]; + maxTo[0] = ranges[0].to; + for (int i = 1; i < ranges.length; ++i) { + maxTo[i] = Math.max(ranges[i].to, maxTo[i - 1]); + } + } + + private final double[] maxTo; + + @Override + protected int collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, int lowBound) throws IOException { + int lo = lowBound, hi = ranges.length - 1; // all candidates are between these indexes + int mid = (lo + hi) >>> 1; + while (lo <= hi) { + if (value < ranges[mid].from) { + hi = mid - 1; + } else if (value >= maxTo[mid]) { + lo = mid + 1; + } else { + break; + } + mid = (lo + hi) >>> 1; + } + if (lo > hi) return lo; // no potential candidate + + // binary search the lower bound + int startLo = lo, startHi = mid; + while (startLo <= startHi) { + final int startMid = (startLo + startHi) >>> 1; + if (value >= maxTo[startMid]) { + startLo = startMid + 1; + } else { + startHi = startMid - 1; + } + } + + // binary search the upper bound + int endLo = mid, endHi = hi; + while (endLo <= endHi) { + final int endMid = (endLo + endHi) >>> 1; + if (value < ranges[endMid].from) { + endHi = endMid - 1; + } else { + endLo = endMid + 1; + } + } + + assert startLo == lowBound || value >= maxTo[startLo - 1]; + assert endHi == ranges.length - 1 || value < ranges[endHi + 1].from; + + for (int i = startLo; i <= endHi; ++i) { + if (ranges[i].matches(value)) { + collectBucket(sub, doc, subBucketOrdinal(owningBucketOrdinal, i)); + } + } + + // The next value must fall in the next bucket to be collected. + return endHi + 1; + } + } + + private static class FromFilters extends AdaptingAggregator { + private final DocValueFormat format; + private final Range[] ranges; + private final boolean keyed; + private final InternalRange.Factory rangeFactory; + + FromFilters( + Aggregator parent, + AggregatorFactories subAggregators, + CheckedFunction delegate, + DocValueFormat format, + Range[] ranges, + boolean keyed, + InternalRange.Factory rangeFactory + ) throws IOException { + super(parent, subAggregators, delegate); + this.format = format; + this.ranges = ranges; + this.keyed = keyed; + this.rangeFactory = rangeFactory; + } + + @Override + protected InternalAggregation adapt(InternalAggregation delegateResult) { + InternalFilters filters = (InternalFilters) delegateResult; + if (filters.getBuckets().size() != ranges.length) { + throw new IllegalStateException( + "bad number of filters [" + filters.getBuckets().size() + "] expecting [" + ranges.length + "]" + ); + } + List buckets = new ArrayList<>(filters.getBuckets().size()); + for (int i = 0; i < ranges.length; i++) { + Range r = ranges[i]; + InternalFilters.InternalBucket b = filters.getBuckets().get(i); + buckets.add( + rangeFactory.createBucket( + r.getKey(), + r.getFrom(), + r.getTo(), + b.getDocCount(), + (InternalAggregations) b.getAggregations(), + keyed, + format + ) + ); + } + return rangeFactory.create(name(), buckets, format, keyed, filters.getMetadata()); + } + } + + private static boolean hasOverlap(Range[] ranges) { + double lastEnd = ranges[0].to; + for (int i = 1; i < ranges.length; ++i) { + if (ranges[i].from < lastEnd) { + return true; + } + lastEnd = ranges[i].to; + } + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java index 4bbfd3050106d..2e6714ee83b3e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java @@ -18,11 +18,10 @@ */ package org.elasticsearch.search.aggregations.bucket.range; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.CardinalityUpperBound; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -31,9 +30,8 @@ public interface RangeAggregatorSupplier { Aggregator build(String name, AggregatorFactories factories, - ValuesSource.Numeric valuesSource, - DocValueFormat format, - InternalRange.Factory rangeFactory, + ValuesSourceConfig valuesSourceConfig, + InternalRange.Factory rangeFactory, RangeAggregator.Range[] ranges, boolean keyed, SearchContext context, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index 5afcd394d645a..bb4386360fc20 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -385,11 +385,17 @@ public boolean hasGlobalOrdinals() { */ @Nullable public Function getPointReaderOrNull() { - MappedFieldType fieldType = fieldType(); - if (fieldType != null && script() == null && missing() == null) { - return fieldType.pointReaderIfPossible(); - } - return null; + return alignesWithSearchIndex() ? fieldType().pointReaderIfPossible() : null; + } + + /** + * Do {@link ValuesSource}s built by this config line up with the search + * index of the underlying field? This'll only return true if the fields + * is searchable and there aren't missing values or a script to confuse + * the ordering. + */ + public boolean alignesWithSearchIndex() { + return script() == null && missing() == null && fieldType() != null && fieldType().isSearchable(); } /** diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java index 44d47ef12245b..5f435c7bc3990 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java +++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java @@ -31,17 +31,7 @@ protected AggregationProfileBreakdown createProfileBreakdown() { @Override protected String getTypeFromElement(Aggregator element) { - - // Anonymous classes (such as NonCollectingAggregator in TermsAgg) won't have a name, - // we need to get the super class - if (element.getClass().getSimpleName().isEmpty()) { - return element.getClass().getSuperclass().getSimpleName(); - } - Class enclosing = element.getClass().getEnclosingClass(); - if (enclosing != null) { - return enclosing.getSimpleName() + "." + element.getClass().getSimpleName(); - } - return element.getClass().getSimpleName(); + return typeFromAggregator(element); } @Override @@ -49,4 +39,16 @@ protected String getDescriptionFromElement(Aggregator element) { return element.name(); } + public static String typeFromAggregator(Aggregator aggregator) { + // Anonymous classes (such as NonCollectingAggregator in TermsAgg) won't have a name, + // we need to get the super class + if (aggregator.getClass().getSimpleName().isEmpty()) { + return aggregator.getClass().getSuperclass().getSimpleName(); + } + Class enclosing = aggregator.getClass().getEnclosingClass(); + if (enclosing != null) { + return enclosing.getSimpleName() + "." + aggregator.getClass().getSimpleName(); + } + return aggregator.getClass().getSimpleName(); + } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java index 83023325ac95f..9c88d387c5628 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java @@ -123,6 +123,11 @@ public String toString() { return delegate.toString(); } + @Override + public Aggregator[] subAggregators() { + return delegate.subAggregators(); + } + public static Aggregator unwrap(Aggregator agg) { if (agg instanceof ProfilingAggregator) { return ((ProfilingAggregator) agg).delegate; diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/gradle.policy b/server/src/main/resources/org/elasticsearch/bootstrap/gradle.policy new file mode 100644 index 0000000000000..eae776d22fbbf --- /dev/null +++ b/server/src/main/resources/org/elasticsearch/bootstrap/gradle.policy @@ -0,0 +1,16 @@ +grant codeBase "file:${gradle.dist.lib}/-" { + // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production + // dependency and there's no point in exercising the security policy against it + permission java.security.AllPermission; +}; + +grant codeBase "file:${gradle.worker.jar}" { + // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production + // dependency and there's no point in exercising the security policy against it + permission java.security.AllPermission; +}; + +grant { + // since the gradle test worker jar is on the test classpath, our tests should be able to read it + permission java.io.FilePermission "${gradle.worker.jar}", "read"; +}; diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/intellij.policy b/server/src/main/resources/org/elasticsearch/bootstrap/intellij.policy new file mode 100644 index 0000000000000..42448c51ec68d --- /dev/null +++ b/server/src/main/resources/org/elasticsearch/bootstrap/intellij.policy @@ -0,0 +1,6 @@ +grant codeBase "${codebase.junit-rt.jar}" { + // allows IntelliJ IDEA JUnit test runner to control number of test iterations + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + + diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index b195662dcf20e..4a8647e5a0474 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -88,25 +88,3 @@ grant codeBase "${codebase.httpasyncclient}" { // rest client uses system properties which gets the default proxy permission java.net.NetPermission "getProxySelector"; }; - -grant codeBase "${codebase.junit-rt.jar}" { - // allows IntelliJ IDEA JUnit test runner to control number of test iterations - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; -}; - -grant codeBase "file:${gradle.dist.lib}/-" { - // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production - // dependency and there's no point in exercising the security policy against it - permission java.security.AllPermission; -}; - -grant codeBase "file:${gradle.worker.jar}" { - // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production - // dependency and there's no point in exercising the security policy against it - permission java.security.AllPermission; -}; - -grant { - // since the gradle test worker jar is on the test classpath, our tests should be able to read it - permission java.io.FilePermission "${gradle.worker.jar}", "read"; -}; diff --git a/server/src/test/java/org/elasticsearch/common/RoundingTests.java b/server/src/test/java/org/elasticsearch/common/RoundingTests.java index fa94cacbbe77c..5e7392caf9fdc 100644 --- a/server/src/test/java/org/elasticsearch/common/RoundingTests.java +++ b/server/src/test/java/org/elasticsearch/common/RoundingTests.java @@ -39,9 +39,11 @@ import java.time.zone.ZoneOffsetTransitionRule; import java.time.zone.ZoneRules; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; +import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -1017,6 +1019,35 @@ public void testNonMillisecondsBasedUnitCalendarRoundingSize() { assertThat(prepared.roundingSize(thirdQuarter, Rounding.DateTimeUnit.HOUR_OF_DAY), closeTo(2208.0, 0.000001)); } + public void testFixedRoundingPoints() { + Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.QUARTER_OF_YEAR).build(); + assertFixedRoundingPoints( + rounding.prepare(time("2020-01-01T00:00:00"), time("2021-01-01T00:00:00")), + "2020-01-01T00:00:00", + "2020-04-01T00:00:00", + "2020-07-01T00:00:00", + "2020-10-01T00:00:00", + "2021-01-01T00:00:00" + ); + rounding = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build(); + assertFixedRoundingPoints( + rounding.prepare(time("2020-01-01T00:00:00"), time("2020-01-06T00:00:00")), + "2020-01-01T00:00:00", + "2020-01-02T00:00:00", + "2020-01-03T00:00:00", + "2020-01-04T00:00:00", + "2020-01-05T00:00:00", + "2020-01-06T00:00:00" + ); + } + + private void assertFixedRoundingPoints(Rounding.Prepared prepared, String... expected) { + assertThat( + Arrays.stream(prepared.fixedRoundingPoints()).mapToObj(Instant::ofEpochMilli).collect(toList()), + equalTo(Arrays.stream(expected).map(RoundingTests::time).map(Instant::ofEpochMilli).collect(toList())) + ); + } + private void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes, ZoneId tz) { assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz); diff --git a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java index 5dbe2aafedd58..448aa96a695ae 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java @@ -19,12 +19,9 @@ package org.elasticsearch.common.util; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.breaker.CircuitBreakingException; -import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -33,8 +30,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.junit.Assume.assumeThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class BitArrayTests extends ESTestCase { @@ -89,32 +84,17 @@ public void testTooBigIsNotSet() { } public void testClearingDoesntAllocate() { - CircuitBreakerService breaker = mock(CircuitBreakerService.class); ByteSizeValue max = new ByteSizeValue(1, ByteSizeUnit.KB); - when(breaker.getBreaker(CircuitBreaker.REQUEST)).thenReturn(new NoopCircuitBreaker(CircuitBreaker.REQUEST) { - private long total = 0; - - @Override - public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException { - total += bytes; - if (total > max.getBytes()) { - throw new CircuitBreakingException("test error", bytes, max.getBytes(), Durability.TRANSIENT); - } - return total; - } - - @Override - public long addWithoutBreaking(long bytes) { - total += bytes; - return total; - } - }); - BigArrays bigArrays = new BigArrays(null, breaker, CircuitBreaker.REQUEST, true); + MockBigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), max); try (BitArray bitArray = new BitArray(1, bigArrays)) { bitArray.clear(100000000); } } + public void testAllocation() { + MockBigArrays.assertFitsIn(new ByteSizeValue(100), bigArrays -> new BitArray(1, bigArrays)); + } + public void testOr() { try (BitArray bitArray1 = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE); BitArray bitArray2 = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE); diff --git a/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java b/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java index b435ca8fd0b8c..d8a9064f3cba7 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; @@ -36,62 +37,51 @@ import java.util.Set; public class BytesRefHashTests extends ESTestCase { - - BytesRefHash hash; - - private BigArrays randomBigArrays() { + private BigArrays mockBigArrays() { return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); } - private void newHash() { - if (hash != null) { - hash.close(); - } + private BytesRefHash randomHash() { // Test high load factors to make sure that collision resolution works fine final float maxLoadFactor = 0.6f + randomFloat() * 0.39f; - hash = new BytesRefHash(randomIntBetween(0, 100), maxLoadFactor, randomBigArrays()); - } - - @Override - public void setUp() throws Exception { - super.setUp(); - newHash(); + return new BytesRefHash(randomIntBetween(0, 100), maxLoadFactor, mockBigArrays()); } public void testDuel() { - final int len = randomIntBetween(1, 100000); - final BytesRef[] values = new BytesRef[len]; - for (int i = 0; i < values.length; ++i) { - values[i] = new BytesRef(randomAlphaOfLength(5)); - } - final ObjectLongMap valueToId = new ObjectLongHashMap<>(); - final BytesRef[] idToValue = new BytesRef[values.length]; - final int iters = randomInt(1000000); - for (int i = 0; i < iters; ++i) { - final BytesRef value = randomFrom(values); - if (valueToId.containsKey(value)) { - assertEquals(- 1 - valueToId.get(value), hash.add(value, value.hashCode())); - } else { - assertEquals(valueToId.size(), hash.add(value, value.hashCode())); - idToValue[valueToId.size()] = value; - valueToId.put(value, valueToId.size()); + try (BytesRefHash hash = randomHash()) { + final int len = randomIntBetween(1, 100000); + final BytesRef[] values = new BytesRef[len]; + for (int i = 0; i < values.length; ++i) { + values[i] = new BytesRef(randomAlphaOfLength(5)); + } + final ObjectLongMap valueToId = new ObjectLongHashMap<>(); + final BytesRef[] idToValue = new BytesRef[values.length]; + final int iters = randomInt(1000000); + for (int i = 0; i < iters; ++i) { + final BytesRef value = randomFrom(values); + if (valueToId.containsKey(value)) { + assertEquals(- 1 - valueToId.get(value), hash.add(value, value.hashCode())); + } else { + assertEquals(valueToId.size(), hash.add(value, value.hashCode())); + idToValue[valueToId.size()] = value; + valueToId.put(value, valueToId.size()); + } } - } - assertEquals(valueToId.size(), hash.size()); - for (final ObjectLongCursor next : valueToId) { - assertEquals(next.value, hash.find(next.key, next.key.hashCode())); - } + assertEquals(valueToId.size(), hash.size()); + for (final ObjectLongCursor next : valueToId) { + assertEquals(next.value, hash.find(next.key, next.key.hashCode())); + } - for (long i = 0; i < hash.capacity(); ++i) { - final long id = hash.id(i); - BytesRef spare = new BytesRef(); - if (id >= 0) { - hash.get(id, spare); - assertEquals(idToValue[(int) id], spare); + for (long i = 0; i < hash.capacity(); ++i) { + final long id = hash.id(i); + BytesRef spare = new BytesRef(); + if (id >= 0) { + hash.get(id, spare); + assertEquals(idToValue[(int) id], spare); + } } } - hash.close(); } // START - tests borrowed from LUCENE @@ -100,6 +90,7 @@ public void testDuel() { * Test method for {@link org.apache.lucene.util.BytesRefHash#size()}. */ public void testSize() { + BytesRefHash hash = randomHash(); BytesRefBuilder ref = new BytesRefBuilder(); int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { @@ -112,12 +103,14 @@ public void testSize() { ref.copyChars(str); long count = hash.size(); long key = hash.add(ref.get()); - if (key < 0) + if (key < 0) { assertEquals(hash.size(), count); - else + } else { assertEquals(hash.size(), count + 1); + } if(i % mod == 0) { - newHash(); + hash.close(); + hash = randomHash(); } } } @@ -130,6 +123,7 @@ public void testSize() { * . */ public void testGet() { + BytesRefHash hash = randomHash(); BytesRefBuilder ref = new BytesRefBuilder(); BytesRef scratch = new BytesRef(); int num = scaledRandomIntBetween(2, 20); @@ -158,7 +152,8 @@ public void testGet() { ref.copyChars(entry.getKey()); assertEquals(ref.get(), hash.get(entry.getValue(), scratch)); } - newHash(); + hash.close(); + hash = randomHash(); } hash.close(); } @@ -169,6 +164,7 @@ public void testGet() { * . */ public void testAdd() { + BytesRefHash hash = randomHash(); BytesRefBuilder ref = new BytesRefBuilder(); BytesRef scratch = new BytesRef(); int num = scaledRandomIntBetween(2, 20); @@ -198,12 +194,14 @@ public void testAdd() { } assertAllIn(strings, hash); - newHash(); + hash.close(); + hash = randomHash(); } hash.close(); } public void testFind() { + BytesRefHash hash = randomHash(); BytesRefBuilder ref = new BytesRefBuilder(); BytesRef scratch = new BytesRef(); int num = scaledRandomIntBetween(2, 20); @@ -233,7 +231,8 @@ public void testFind() { } assertAllIn(strings, hash); - newHash(); + hash.close(); + hash = randomHash(); } hash.close(); } @@ -254,4 +253,7 @@ private void assertAllIn(Set strings, BytesRefHash hash) { // END - tests borrowed from LUCENE + public void testAllocation() { + MockBigArrays.assertFitsIn(new ByteSizeValue(512), bigArrays -> new BytesRefHash(1, bigArrays)); + } } diff --git a/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java b/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java index 31b499d5c4984..a733f9b6b8ffc 100644 --- a/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java @@ -24,6 +24,7 @@ import com.carrotsearch.hppc.cursors.LongLongCursor; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; @@ -34,68 +35,57 @@ import java.util.Set; public class LongHashTests extends ESTestCase { - LongHash hash; - - private BigArrays randombigArrays() { + private BigArrays mockBigArrays() { return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); } - private void newHash() { - if (hash != null) { - hash.close(); - } - + private LongHash randomHash() { // Test high load factors to make sure that collision resolution works fine - final float maxLoadFactor = 0.6f + randomFloat() * 0.39f; - hash = new LongHash(randomIntBetween(0, 100), maxLoadFactor, randombigArrays()); - } - - @Override - public void setUp() throws Exception { - super.setUp(); - newHash(); + float maxLoadFactor = 0.6f + randomFloat() * 0.39f; + return new LongHash(randomIntBetween(0, 100), maxLoadFactor, mockBigArrays()); } public void testDuell() { - final Long[] values = new Long[randomIntBetween(1, 100000)]; - for (int i = 0; i < values.length; ++i) { - values[i] = randomLong(); - } - final LongLongMap valueToId = new LongLongHashMap(); - final long[] idToValue = new long[values.length]; - final int iters = randomInt(1000000); - for (int i = 0; i < iters; ++i) { - final Long value = randomFrom(values); - if (valueToId.containsKey(value)) { - assertEquals(-1 - valueToId.get(value), hash.add(value)); - } else { - assertEquals(valueToId.size(), hash.add(value)); - idToValue[valueToId.size()] = value; - valueToId.put(value, valueToId.size()); + try (LongHash hash = randomHash()) { + final Long[] values = new Long[randomIntBetween(1, 100000)]; + for (int i = 0; i < values.length; ++i) { + values[i] = randomLong(); + } + final LongLongMap valueToId = new LongLongHashMap(); + final long[] idToValue = new long[values.length]; + final int iters = randomInt(1000000); + for (int i = 0; i < iters; ++i) { + final Long value = randomFrom(values); + if (valueToId.containsKey(value)) { + assertEquals(-1 - valueToId.get(value), hash.add(value)); + } else { + assertEquals(valueToId.size(), hash.add(value)); + idToValue[valueToId.size()] = value; + valueToId.put(value, valueToId.size()); + } } - } - assertEquals(valueToId.size(), hash.size()); - for (Iterator iterator = valueToId.iterator(); iterator.hasNext(); ) { - final LongLongCursor next = iterator.next(); - assertEquals(next.value, hash.find(next.key)); - } + assertEquals(valueToId.size(), hash.size()); + for (Iterator iterator = valueToId.iterator(); iterator.hasNext(); ) { + final LongLongCursor next = iterator.next(); + assertEquals(next.value, hash.find(next.key)); + } - for (long i = 0; i < hash.capacity(); ++i) { - final long id = hash.id(i); - if (id >= 0) { - assertEquals(idToValue[(int) id], hash.get(id)); + for (long i = 0; i < hash.capacity(); ++i) { + final long id = hash.id(i); + if (id >= 0) { + assertEquals(idToValue[(int) id], hash.get(id)); + } } - } - for (long i = 0; i < hash.size(); i++) { - assertEquals(idToValue[(int) i], hash.get(i)); + for (long i = 0; i < hash.size(); i++) { + assertEquals(idToValue[(int) i], hash.get(i)); + } } - - hash.close(); } public void testSize() { + LongHash hash = randomHash(); int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { final int mod = 1 + randomInt(40); @@ -107,7 +97,8 @@ public void testSize() { else assertEquals(hash.size(), count + 1); if (i % mod == 0) { - newHash(); + hash.close(); + hash = randomHash(); } } } @@ -115,6 +106,7 @@ public void testSize() { } public void testKey() { + LongHash hash = randomHash(); int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { Map longs = new HashMap<>(); @@ -140,12 +132,14 @@ public void testKey() { assertEquals(expected, hash.get(keyIdx)); } - newHash(); + hash.close(); + hash = randomHash(); } hash.close(); } public void testAdd() { + LongHash hash = randomHash(); int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { Set longs = new HashSet<>(); @@ -168,12 +162,14 @@ public void testAdd() { } assertAllIn(longs, hash); - newHash(); + hash.close(); + hash = randomHash(); } hash.close(); } public void testFind() throws Exception { + LongHash hash = randomHash(); int num = scaledRandomIntBetween(2, 20); for (int j = 0; j < num; j++) { Set longs = new HashSet<>(); @@ -197,11 +193,16 @@ public void testFind() throws Exception { } assertAllIn(longs, hash); - newHash(); + hash.close(); + hash = randomHash(); } hash.close(); } + public void testAllocation() { + MockBigArrays.assertFitsIn(new ByteSizeValue(160), bigArrays -> new LongHash(1, bigArrays)); + } + private static void assertAllIn(Set longs, LongHash hash) { long count = hash.size(); for (Long l : longs) { diff --git a/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java b/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java index bdfb6b89ccd4a..9910446cd0fea 100644 --- a/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.util; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; @@ -100,6 +101,10 @@ public void testDuel() { } } + public void testAllocation() { + MockBigArrays.assertFitsIn(new ByteSizeValue(256), bigArrays -> new LongLongHash(1, bigArrays)); + } + class Key { long key1; long key2; diff --git a/server/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java b/server/src/test/java/org/elasticsearch/common/util/LongObjectPagedHashMapTests.java similarity index 87% rename from server/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java rename to server/src/test/java/org/elasticsearch/common/util/LongObjectPagedHashMapTests.java index 02605d35cefdc..758df9b1081a6 100644 --- a/server/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/LongObjectPagedHashMapTests.java @@ -21,19 +21,20 @@ import com.carrotsearch.hppc.LongObjectHashMap; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; -public class LongObjectHashMapTests extends ESTestCase { +public class LongObjectPagedHashMapTests extends ESTestCase { - private BigArrays randomBigArrays() { + private BigArrays mockBigArrays() { return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); } public void testDuel() { final LongObjectHashMap map1 = new LongObjectHashMap<>(); final LongObjectPagedHashMap map2 = - new LongObjectPagedHashMap<>(randomInt(42), 0.6f + randomFloat() * 0.39f, randomBigArrays()); + new LongObjectPagedHashMap<>(randomInt(42), 0.6f + randomFloat() * 0.39f, mockBigArrays()); final int maxKey = randomIntBetween(1, 10000); final int iters = scaledRandomIntBetween(10000, 100000); for (int i = 0; i < iters; ++i) { @@ -61,4 +62,8 @@ public void testDuel() { assertEquals(map1, copy); } + public void testAllocation() { + MockBigArrays.assertFitsIn(new ByteSizeValue(256), bigArrays -> new LongObjectPagedHashMap(1, bigArrays)); + } + } diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 7f18327f929f2..129bde0b45d76 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -55,6 +55,7 @@ import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortedSetSortField; @@ -108,6 +109,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.fieldvisitor.FieldsVisitor; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; @@ -123,6 +125,7 @@ import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.SearcherHelper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; import org.elasticsearch.index.store.Store; @@ -695,8 +698,7 @@ public void testConcurrentGetAndFlush() throws Exception { engine.index(indexForDoc(doc)); final AtomicReference latestGetResult = new AtomicReference<>(); - final BiFunction searcherFactory = engine::acquireSearcher; - latestGetResult.set(engine.get(newGet(true, doc), searcherFactory)); + latestGetResult.set(engine.get(newGet(true, doc), docMapper(), randomSearcherWrapper())); final AtomicBoolean flushFinished = new AtomicBoolean(false); final CyclicBarrier barrier = new CyclicBarrier(2); Thread getThread = new Thread(() -> { @@ -710,7 +712,7 @@ public void testConcurrentGetAndFlush() throws Exception { if (previousGetResult != null) { previousGetResult.close(); } - latestGetResult.set(engine.get(newGet(true, doc), searcherFactory)); + latestGetResult.set(engine.get(newGet(true, doc), docMapper(), randomSearcherWrapper())); if (latestGetResult.get().exists() == false) { break; } @@ -726,6 +728,7 @@ public void testConcurrentGetAndFlush() throws Exception { } public void testSimpleOperations() throws Exception { + final DocumentMapper mapper = docMapper(); engine.refresh("warm_up"); Engine.Searcher searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); @@ -747,18 +750,18 @@ public void testSimpleOperations() throws Exception { searchResult.close(); // but, not there non realtime - try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) { + try (Engine.GetResult getResult = engine.get(newGet(false, doc), mapper, randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(false)); } // but, we can still get it (in realtime) - try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) { + try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); } // but not real time is not yet visible - try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) { + try (Engine.GetResult getResult = engine.get(newGet(false, doc), mapper, randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(false)); } @@ -773,7 +776,7 @@ public void testSimpleOperations() throws Exception { searchResult.close(); // also in non realtime - try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) { + try (Engine.GetResult getResult = engine.get(newGet(false, doc), mapper, randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); } @@ -781,8 +784,8 @@ public void testSimpleOperations() throws Exception { // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", null, document, B_2, null); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(SOURCE), SourceFieldMapper.Defaults.FIELD_TYPE)); + doc = testParsedDocument("1", null, document, SOURCE, null); engine.index(indexForDoc(doc)); // its not updated yet... @@ -795,7 +798,7 @@ public void testSimpleOperations() throws Exception { searchResult.close(); // but, we can still get it (in realtime) - try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) { + try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); } @@ -824,7 +827,7 @@ public void testSimpleOperations() throws Exception { searchResult.close(); // but, get should not see it (in realtime) - try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) { + try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(false)); } @@ -870,7 +873,7 @@ public void testSimpleOperations() throws Exception { engine.flush(); // and, verify get (in real time) - try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) { + try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); } @@ -903,6 +906,53 @@ public void testSimpleOperations() throws Exception { searchResult.close(); } + public void testGetWithSearcherWrapper() throws Exception { + engine.refresh("warm_up"); + engine.index(indexForDoc(createParsedDoc("1", null))); + assertThat(engine.lastRefreshedCheckpoint(), equalTo(NO_OPS_PERFORMED)); + try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), randomSearcherWrapper())) { + // we do not track the translog location yet + assertTrue(get.exists()); + assertFalse(get.isFromTranslog()); + } + // refresh triggered, as we did not track translog location until the first realtime get. + assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L)); + + engine.index(indexForDoc(createParsedDoc("1", null))); + try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), searcher -> searcher)) { + assertTrue(get.exists()); + assertTrue(get.isFromTranslog()); + } + assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L)); // no refresh; just read from translog + if (randomBoolean()) { + engine.index(indexForDoc(createParsedDoc("1", null))); + } + try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), + searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new MatchAllDocsQuery())))) { + assertTrue(get.exists()); + assertFalse(get.isFromTranslog()); + } + + try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), + searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new MatchNoDocsQuery())))) { + assertFalse(get.exists()); + assertFalse(get.isFromTranslog()); + } + + try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), + searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new TermQuery(newUid("1")))))) { + assertTrue(get.exists()); + assertFalse(get.isFromTranslog()); + } + + try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), + searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new TermQuery(newUid("2")))))) { + assertFalse(get.exists()); + assertFalse(get.isFromTranslog()); + } + assertThat("no refresh, just read from translog or in-memory segment", engine.lastRefreshedCheckpoint(), equalTo(0L)); + } + public void testSearchResultRelease() throws Exception { engine.refresh("warm_up"); Engine.Searcher searchResult = engine.acquireSearcher("test"); @@ -1156,7 +1206,7 @@ public void testVersionedUpdate() throws IOException { Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED); Engine.IndexResult indexResult = engine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); - try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) { + try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) { assertEquals(1, get.version()); } @@ -1164,7 +1214,7 @@ public void testVersionedUpdate() throws IOException { Engine.IndexResult update_1_result = engine.index(update_1); assertThat(update_1_result.getVersion(), equalTo(2L)); - try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) { + try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) { assertEquals(2, get.version()); } @@ -1172,14 +1222,13 @@ public void testVersionedUpdate() throws IOException { Engine.IndexResult update_2_result = engine.index(update_2); assertThat(update_2_result.getVersion(), equalTo(3L)); - try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) { + try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) { assertEquals(3, get.version()); } } public void testGetIfSeqNoIfPrimaryTerm() throws IOException { - final BiFunction searcherFactory = engine::acquireSearcher; ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED); @@ -1193,22 +1242,22 @@ public void testGetIfSeqNoIfPrimaryTerm() throws IOException { try (Engine.GetResult get = engine.get( new Engine.Get(true, true, doc.id()) .setIfSeqNo(indexResult.getSeqNo()).setIfPrimaryTerm(primaryTerm.get()), - searcherFactory)) { + docMapper(), randomSearcherWrapper())) { assertEquals(indexResult.getSeqNo(), get.docIdAndVersion().seqNo); } expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.id()) .setIfSeqNo(indexResult.getSeqNo() + 1).setIfPrimaryTerm(primaryTerm.get()), - searcherFactory)); + docMapper(), randomSearcherWrapper())); expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.id()) .setIfSeqNo(indexResult.getSeqNo()).setIfPrimaryTerm(primaryTerm.get() + 1), - searcherFactory)); + docMapper(), randomSearcherWrapper())); final VersionConflictEngineException versionConflictEngineException = expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.id()) .setIfSeqNo(indexResult.getSeqNo() + 1).setIfPrimaryTerm(primaryTerm.get() + 1), - searcherFactory)); + docMapper(), randomSearcherWrapper())); assertThat(versionConflictEngineException.getStackTrace(), emptyArray()); } @@ -1954,7 +2003,6 @@ class OpAndVersion { ParsedDocument doc = testParsedDocument("1", null, testDocument(), bytesArray(""), null); final Term uidTerm = newUid(doc); engine.index(indexForDoc(doc)); - final BiFunction searcherFactory = engine::acquireSearcher; for (int i = 0; i < thread.length; i++) { thread[i] = new Thread(() -> { startGun.countDown(); @@ -1964,7 +2012,7 @@ class OpAndVersion { throw new AssertionError(e); } for (int op = 0; op < opsPerThread; op++) { - try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) { + try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) { FieldsVisitor visitor = new FieldsVisitor(true); get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); List values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString())); @@ -2006,7 +2054,7 @@ class OpAndVersion { assertTrue(op.added + " should not exist", exists); } - try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) { + try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) { FieldsVisitor visitor = new FieldsVisitor(true); get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); List values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString())); @@ -2402,7 +2450,7 @@ public void testEnableGcDeletes() throws Exception { Engine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), null))) { engine.config().setEnableGcDeletes(false); - final BiFunction searcherFactory = engine::acquireSearcher; + final DocumentMapper mapper = docMapper(); // Add document Document document = testDocument(); @@ -2418,7 +2466,7 @@ public void testEnableGcDeletes() throws Exception { 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0)); // Get should not find the document - Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory); + Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper()); assertThat(getResult.exists(), equalTo(false)); // Give the gc pruning logic a chance to kick in @@ -2433,7 +2481,7 @@ public void testEnableGcDeletes() throws Exception { 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0)); // Get should not find the document (we never indexed uid=2): - getResult = engine.get(new Engine.Get(true, false, "2"), searcherFactory); + getResult = engine.get(new Engine.Get(true, false, "2"), mapper, randomSearcherWrapper()); assertThat(getResult.exists(), equalTo(false)); // Try to index uid=1 with a too-old version, should fail: @@ -2444,7 +2492,7 @@ public void testEnableGcDeletes() throws Exception { assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); // Get should still not find the document - getResult = engine.get(newGet(true, doc), searcherFactory); + getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper()); assertThat(getResult.exists(), equalTo(false)); // Try to index uid=2 with a too-old version, should fail: @@ -2455,7 +2503,7 @@ public void testEnableGcDeletes() throws Exception { assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); // Get should not find the document - getResult = engine.get(newGet(true, doc), searcherFactory); + getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper()); assertThat(getResult.exists(), equalTo(false)); } } @@ -3968,7 +4016,7 @@ public void testOutOfOrderSequenceNumbersWithVersionConflict() throws IOExceptio } assertThat(engine.getProcessedLocalCheckpoint(), equalTo(expectedLocalCheckpoint)); - try (Engine.GetResult result = engine.get(new Engine.Get(true, false, "1"), searcherFactory)) { + try (Engine.GetResult result = engine.get(new Engine.Get(true, false, "1"), docMapper(), randomSearcherWrapper())) { assertThat(result.exists(), equalTo(exists)); } } @@ -4851,7 +4899,7 @@ public void testStressShouldPeriodicallyFlush() throws Exception { } public void testStressUpdateSameDocWhileGettingIt() throws IOException, InterruptedException { - final int iters = randomIntBetween(1, 15); + final int iters = randomIntBetween(1, 1); for (int i = 0; i < iters; i++) { // this is a reproduction of https://github.com/elastic/elasticsearch/issues/28714 try (Store store = createStore(); InternalEngine engine = createEngine(store, createTempDir())) { @@ -4893,13 +4941,15 @@ public void testStressUpdateSameDocWhileGettingIt() throws IOException, Interrup CountDownLatch awaitStarted = new CountDownLatch(1); Thread thread = new Thread(() -> { awaitStarted.countDown(); - try (Engine.GetResult getResult = engine.get(new Engine.Get(true, false, doc3.id()), engine::acquireSearcher)) { + try (Engine.GetResult getResult = engine.get( + new Engine.Get(true, false, doc3.id()), docMapper(), searcher -> searcher)) { assertTrue(getResult.exists()); } }); thread.start(); awaitStarted.await(); - try (Engine.GetResult getResult = engine.get(new Engine.Get(true, false, doc.id()), engine::acquireSearcher)) { + try (Engine.GetResult getResult = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), + searcher -> SearcherHelper.wrapSearcher(searcher, r -> new MatchingDirectoryReader(r, new MatchAllDocsQuery())))) { assertFalse(getResult.exists()); } thread.join(); @@ -5848,7 +5898,7 @@ public void afterRefresh(boolean didRefresh) { int iters = randomIntBetween(1, 10); for (int i = 0; i < iters; i++) { ParsedDocument doc = createParsedDoc(randomFrom(ids), null); - try (Engine.GetResult getResult = engine.get(newGet(true, doc), engine::acquireSearcher)) { + try (Engine.GetResult getResult = engine.get(newGet(true, doc), docMapper(), randomSearcherWrapper())) { assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java index 563bd0acc2751..a9731539fc446 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java @@ -100,7 +100,7 @@ public void testReadOnlyEngine() throws Exception { assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint())); assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo())); assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds)); - try (Engine.GetResult getResult = readOnlyEngine.get(get, readOnlyEngine::acquireSearcher)) { + try (Engine.GetResult getResult = readOnlyEngine.get(get, docMapper(), randomSearcherWrapper())) { assertTrue(getResult.exists()); } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index b6e72863279c0..ce108017d6028 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.engine.InternalEngine; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.ParsedDocument; @@ -344,7 +345,8 @@ public void testLotsOfThreads() throws Exception { listener.assertNoError(); Engine.Get get = new Engine.Get(false, false, threadId); - try (Engine.GetResult getResult = engine.get(get, engine::acquireSearcher)) { + final DocumentMapper mapper = EngineTestCase.docMapper(); + try (Engine.GetResult getResult = engine.get(get, mapper, EngineTestCase.randomSearcherWrapper())) { assertTrue("document not found", getResult.exists()); assertEquals(iteration, getResult.version()); org.apache.lucene.document.Document document = diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index d3bde9d174b1a..44a316242b98f 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.RoutingFieldMapper; @@ -117,7 +118,7 @@ private void runGetFromTranslogWithOptions(String docToIndex, String sourceOptio "\"bar\": { \"type\": " + fieldType + "}}, \"_source\": { " + sourceOptions + "}}}") .settings(settings) .primaryTerm(0, 1).build(); - IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, null); + IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, EngineTestCase.randomReaderWrapper()); recoverShardFromStore(primary); Engine.IndexResult test = indexDoc(primary, "test", "0", docToIndex); assertTrue(primary.getEngine().refreshNeeded()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AdaptingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AdaptingAggregatorTests.java new file mode 100644 index 0000000000000..fe9bf2c6fe141 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AdaptingAggregatorTests.java @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.search.aggregations.bucket.histogram.SizedBucketAggregator; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AdaptingAggregatorTests extends MapperServiceTestCase { + /** + * Its important that sub-aggregations of the {@linkplain AdaptingAggregator} + * receive a reference to the {@linkplain AdaptingAggregator} as the parent. + * Without it we can't do things like implement {@link SizedBucketAggregator}. + */ + public void testParent() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> {})); + ValuesSourceRegistry.Builder registry = new ValuesSourceRegistry.Builder(); + MaxAggregationBuilder.registerAggregators(registry); + withAggregationContext(registry.build(), mapperService, List.of(), null, context -> { + SearchContext searchContext = mock(SearchContext.class); + when(searchContext.bigArrays()).thenReturn(context.bigArrays()); + AggregatorFactories.Builder sub = AggregatorFactories.builder(); + sub.addAggregator(new MaxAggregationBuilder("test").field("foo")); + AggregatorFactory factory = new DummyAdaptingAggregatorFactory("test", context, null, sub, null); + Aggregator adapting = factory.create(searchContext, null, CardinalityUpperBound.ONE); + assertThat(adapting.subAggregators()[0].parent(), sameInstance(adapting)); + }); + } + + public void testBuildCallsAdapt() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> {})); + withAggregationContext(mapperService, List.of(), context -> { + SearchContext searchContext = mock(SearchContext.class); + when(searchContext.bigArrays()).thenReturn(context.bigArrays()); + AggregatorFactory factory = new DummyAdaptingAggregatorFactory("test", context, null, AggregatorFactories.builder(), null); + Aggregator adapting = factory.create(searchContext, null, CardinalityUpperBound.ONE); + assertThat(adapting.buildEmptyAggregation().getMetadata(), equalTo(Map.of("dog", "woof"))); + assertThat(adapting.buildTopLevel().getMetadata(), equalTo(Map.of("dog", "woof"))); + }); + } + + private static class DummyAdaptingAggregatorFactory extends AggregatorFactory { + DummyAdaptingAggregatorFactory( + String name, + AggregationContext context, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metadata + ) throws IOException { + super(name, context, parent, subFactoriesBuilder, metadata); + } + + @Override + protected Aggregator createInternal( + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + return new DummyAdaptingAggregator( + parent, + factories, + subAggs -> new DummyAggregator(name, subAggs, context, parent, CardinalityUpperBound.ONE, metadata) + ); + } + } + + private static class DummyAdaptingAggregator extends AdaptingAggregator { + DummyAdaptingAggregator( + Aggregator parent, + AggregatorFactories subAggregators, + CheckedFunction delegate + ) throws IOException { + super(parent, subAggregators, delegate); + } + + @Override + protected InternalAggregation adapt(InternalAggregation delegateResult) { + InternalAggregation result = mock(InternalAggregation.class); + when(result.getMetadata()).thenReturn(Map.of("dog", "woof")); + return result; + } + } + + private static class DummyAggregator extends AggregatorBase { + protected DummyAggregator( + String name, + AggregatorFactories factories, + SearchContext context, + Aggregator parent, + CardinalityUpperBound subAggregatorCardinality, + Map metadata + ) throws IOException { + super(name, factories, context, parent, subAggregatorCardinality, metadata); + } + + @Override + protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + return new InternalAggregation[] {null}; + } + + @Override + public InternalAggregation buildEmptyAggregation() { + // TODO Auto-generated method stub + return null; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index 617eec9799a4d..a36fff08e1a4e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -20,24 +20,35 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.junit.Before; +import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + public class FiltersAggregatorTests extends AggregatorTestCase { private MappedFieldType fieldType; @@ -139,7 +150,7 @@ public void testRandom() throws Exception { // make sure we have more than one segment to test the merge indexWriter.commit(); } - int value = randomInt(maxTerm-1); + int value = randomInt(maxTerm - 1); expectedBucketCount[value] += 1; document.add(new Field("field", Integer.toString(value), KeywordFieldMapper.Defaults.FIELD_TYPE)); indexWriter.addDocument(document); @@ -188,4 +199,25 @@ public void testRandom() throws Exception { directory.close(); } } + + public void testMergePointRangeQueries() throws IOException { + MappedFieldType ft = new DateFieldMapper.DateFieldType("test", Resolution.MILLISECONDS); + AggregationBuilder builder = new FiltersAggregationBuilder( + "test", + new KeyedFilter("q1", new RangeQueryBuilder("test").from("2020-01-01").to("2020-03-01").includeUpper(false)) + ); + Query query = LongPoint.newRangeQuery( + "test", + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-01-01"), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-02-01") + ); + testCase(builder, query, iw -> { + iw.addDocument(List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2010-01-02")))); + iw.addDocument(List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-01-02")))); + }, result -> { + InternalFilters filters = (InternalFilters) result; + assertThat(filters.getBuckets(), hasSize(1)); + assertThat(filters.getBucketByKey("q1").getDocCount(), equalTo(1L)); + }, ft); + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQueryTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQueryTests.java new file mode 100644 index 0000000000000..d24e756d422f2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQueryTests.java @@ -0,0 +1,249 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.filter; + +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.nullValue; + +public class MergedPointRangeQueryTests extends ESTestCase { + public void testDifferentField() { + assertThat(merge(LongPoint.newExactQuery("a", 0), LongPoint.newExactQuery("b", 0)), nullValue()); + } + + public void testDifferentDimensionCount() { + assertThat( + merge(LongPoint.newExactQuery("a", 0), LongPoint.newRangeQuery("a", new long[] { 1, 2 }, new long[] { 1, 2 })), + nullValue() + ); + } + + public void testDifferentDimensionSize() { + assertThat(merge(LongPoint.newExactQuery("a", 0), IntPoint.newExactQuery("a", 0)), nullValue()); + } + + public void testSame() { + Query lhs = LongPoint.newRangeQuery("a", 0, 100); + assertThat(merge(lhs, LongPoint.newRangeQuery("a", 0, 100)), equalTo(lhs)); + } + + public void testOverlap() throws IOException { + MergedPointRangeQuery overlapping = mergeToMergedQuery( + LongPoint.newRangeQuery("a", -100, 100), + LongPoint.newRangeQuery("a", 0, 100) + ); + assertDelegateForSingleValuedSegmentsEqualPointRange(overlapping, LongPoint.newRangeQuery("a", 0, 100)); + assertFalse(matches1d(overlapping, -50)); // Point not in range + assertTrue(matches1d(overlapping, 50)); // Point in range + assertTrue(matches1d(overlapping, -50, 10)); // Both points in range matches the doc + assertTrue(matches1d(overlapping, -200, 50)); // One point in range matches + assertFalse(matches1d(overlapping, -50, 200)); // No points in range doesn't match + } + + public void testNonOverlap() throws IOException { + MergedPointRangeQuery disjoint = mergeToMergedQuery(LongPoint.newRangeQuery("a", -100, -10), LongPoint.newRangeQuery("a", 10, 100)); + assertThat(disjoint.delegateForSingleValuedSegments(), instanceOf(MatchNoDocsQuery.class)); + assertFalse(matches1d(disjoint, randomLong())); // No single point can match + assertFalse(matches1d(disjoint, -50, -20)); // Both points in lower + assertFalse(matches1d(disjoint, 20, 50)); // Both points in upper + assertTrue(matches1d(disjoint, -50, 50)); // One in lower, one in upper + assertFalse(matches1d(disjoint, -50, 200)); // No point in lower + assertFalse(matches1d(disjoint, -200, 50)); // No point in upper + } + + public void test2dSimpleOverlap() throws IOException { + MergedPointRangeQuery overlapping = mergeToMergedQuery( + LongPoint.newRangeQuery("a", new long[] { -100, -100 }, new long[] { 100, 100 }), + LongPoint.newRangeQuery("a", new long[] { 0, 0 }, new long[] { 100, 100 }) + ); + assertDelegateForSingleValuedSegmentsEqualPointRange( + overlapping, + LongPoint.newRangeQuery("a", new long[] { 0, 0 }, new long[] { 100, 100 }) + ); + assertFalse(matches2d(overlapping, -50, -50)); + assertTrue(matches2d(overlapping, 10, 10)); + assertTrue(matches2d(overlapping, -50, -50, 10, 10)); + } + + public void test2dComplexOverlap() throws IOException { + MergedPointRangeQuery overlapping = mergeToMergedQuery( + LongPoint.newRangeQuery("a", new long[] { -100, 0 }, new long[] { 100, 100 }), + LongPoint.newRangeQuery("a", new long[] { 0, -100 }, new long[] { 100, 100 }) + ); + assertDelegateForSingleValuedSegmentsEqualPointRange( + overlapping, + LongPoint.newRangeQuery("a", new long[] { 0, 0 }, new long[] { 100, 100 }) + ); + assertFalse(matches2d(overlapping, -50, -50)); + assertTrue(matches2d(overlapping, 10, 10)); + assertTrue(matches2d(overlapping, -50, -50, 10, 10)); + } + + public void test2dNoOverlap() throws IOException { + MergedPointRangeQuery disjoint = mergeToMergedQuery( + LongPoint.newRangeQuery("a", new long[] { -100, -100 }, new long[] { -10, -10 }), + LongPoint.newRangeQuery("a", new long[] { 10, 10 }, new long[] { 100, 100 }) + ); + assertThat(disjoint.delegateForSingleValuedSegments(), instanceOf(MatchNoDocsQuery.class)); + assertFalse(matches2d(disjoint, randomLong(), randomLong())); + assertFalse(matches2d(disjoint, -50, -50)); + assertFalse(matches2d(disjoint, 50, 50)); + assertTrue(matches2d(disjoint, -50, -50, 50, 50)); + } + + public void test2dNoOverlapInOneDimension() throws IOException { + MergedPointRangeQuery disjoint = mergeToMergedQuery( + LongPoint.newRangeQuery("a", new long[] { -100, -100 }, new long[] { 100, -10 }), + LongPoint.newRangeQuery("a", new long[] { 0, 10 }, new long[] { 100, 100 }) + ); + assertThat(disjoint.delegateForSingleValuedSegments(), instanceOf(MatchNoDocsQuery.class)); + assertFalse(matches2d(disjoint, randomLong(), randomLong())); + assertFalse(matches2d(disjoint, -50, -50)); + assertFalse(matches2d(disjoint, 50, 50)); + assertTrue(matches2d(disjoint, 50, -50, 50, 50)); + } + + public void testEqualsAndHashCode() { + String field = randomAlphaOfLength(5); + int dims = randomBoolean() ? 1 : between(2, 16); + Supplier supplier = randomFrom( + List.of( + () -> randomIntPointRangequery(field, dims), + () -> randomLongPointRangequery(field, dims), + () -> randomDoublePointRangequery(field, dims) + ) + ); + Query lhs = supplier.get(); + Query rhs = randomValueOtherThan(lhs, supplier); + MergedPointRangeQuery query = mergeToMergedQuery(lhs, rhs); + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + query, + ignored -> mergeToMergedQuery(lhs, rhs), + ignored -> mergeToMergedQuery(lhs, randomValueOtherThan(lhs, () -> randomValueOtherThan(rhs, supplier))) + ); + } + + private Query randomIntPointRangequery(String field, int dims) { + int[] lower = new int[dims]; + int[] upper = new int[dims]; + for (int i = 0; i < dims; i++) { + lower[i] = randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE - 1); + upper[i] = randomIntBetween(lower[i], Integer.MAX_VALUE); + } + return IntPoint.newRangeQuery(field, lower, upper); + } + + private Query randomLongPointRangequery(String field, int dims) { + long[] lower = new long[dims]; + long[] upper = new long[dims]; + for (int i = 0; i < dims; i++) { + lower[i] = randomLongBetween(Long.MIN_VALUE, Long.MAX_VALUE - 1); + upper[i] = randomLongBetween(lower[i], Long.MAX_VALUE); + } + return LongPoint.newRangeQuery(field, lower, upper); + } + + private Query randomDoublePointRangequery(String field, int dims) { + double[] lower = new double[dims]; + double[] upper = new double[dims]; + for (int i = 0; i < dims; i++) { + lower[i] = randomDoubleBetween(Double.MIN_VALUE, 0, true); + upper[i] = randomDoubleBetween(lower[i], Double.MAX_VALUE, true); + } + return DoublePoint.newRangeQuery(field, lower, upper); + } + + private Query merge(Query lhs, Query rhs) { + assertThat("error in test assumptions", lhs, instanceOf(PointRangeQuery.class)); + assertThat("error in test assumptions", rhs, instanceOf(PointRangeQuery.class)); + return MergedPointRangeQuery.merge((PointRangeQuery) lhs, (PointRangeQuery) rhs); + } + + private MergedPointRangeQuery mergeToMergedQuery(Query lhs, Query rhs) { + Query merged = merge(lhs, rhs); + assertThat(merged, instanceOf(MergedPointRangeQuery.class)); + return (MergedPointRangeQuery) merged; + } + + private boolean matches1d(Query query, long... values) throws IOException { + try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) { + List doc = new ArrayList<>(); + for (long v : values) { + doc.add(new LongPoint("a", v)); + } + iw.addDocument(doc); + try (IndexReader r = iw.getReader()) { + IndexSearcher searcher = new IndexSearcher(r); + return searcher.count(query) > 0; + } + } + } + + private boolean matches2d(Query query, long... values) throws IOException { + try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) { + List doc = new ArrayList<>(); + assertEquals(values.length % 2, 0); + for (int i = 0; i < values.length; i += 2) { + doc.add(new LongPoint("a", values[i], values[i + 1])); + } + iw.addDocument(doc); + try (IndexReader r = iw.getReader()) { + IndexSearcher searcher = new IndexSearcher(r); + return searcher.count(query) > 0; + } + } + } + + private void assertDelegateForSingleValuedSegmentsEqualPointRange(MergedPointRangeQuery actual, Query expected) { + /* + * This is a lot like asserThat(actual.delegateForSingleValuedSegments(), equalTo(expected)); but + * that doesn't work because the subclasses aren't the same. + */ + assertThat(expected, instanceOf(PointRangeQuery.class)); + assertThat(actual.delegateForSingleValuedSegments(), instanceOf(PointRangeQuery.class)); + assertThat( + ((PointRangeQuery) actual.delegateForSingleValuedSegments()).getLowerPoint(), + equalTo(((PointRangeQuery) expected).getLowerPoint()) + ); + assertThat( + ((PointRangeQuery) actual.delegateForSingleValuedSegments()).getUpperPoint(), + equalTo(((PointRangeQuery) expected).getUpperPoint()) + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java index e0d644c9fdf18..6f01e4d85c5c0 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -97,9 +98,23 @@ protected final void asSubAggTestCase( } protected final DateFieldMapper.DateFieldType aggregableDateFieldType(boolean useNanosecondResolution, boolean isSearchable) { - return new DateFieldMapper.DateFieldType(AGGREGABLE_DATE, isSearchable, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + return aggregableDateFieldType(useNanosecondResolution, isSearchable, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER); + } + + protected final DateFieldMapper.DateFieldType aggregableDateFieldType( + boolean useNanosecondResolution, + boolean isSearchable, + DateFormatter formatter + ) { + return new DateFieldMapper.DateFieldType( + AGGREGABLE_DATE, + isSearchable, + randomBoolean(), + true, + formatter, useNanosecondResolution ? DateFieldMapper.Resolution.NANOSECONDS : DateFieldMapper.Resolution.MILLISECONDS, - null, Collections.emptyMap()); + null, + Collections.emptyMap() + ); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index d297d9fabc613..5bc00a2e7dc28 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -30,13 +30,17 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.internal.SearchContext; +import org.hamcrest.Matcher; import java.io.IOException; import java.util.ArrayList; @@ -44,9 +48,12 @@ import java.util.Collections; import java.util.List; import java.util.function.Consumer; +import java.util.stream.IntStream; import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCase { /** @@ -1140,6 +1147,84 @@ public void testOverlappingBounds() { "hard bounds: [2010-01-01--2020-01-01], extended bounds: [2009-01-01--2021-01-01]")); } + public void testFewRoundingPointsUsesFromRange() throws IOException { + aggregationImplementationChoiceTestCase( + aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyy")), + IntStream.range(2000, 2010).mapToObj(Integer::toString).collect(toList()), + new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR), + true + ); + } + + public void testManyRoundingPointsDoesNotUseFromRange() throws IOException { + aggregationImplementationChoiceTestCase( + aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyy")), + IntStream.range(2000, 3000).mapToObj(Integer::toString).collect(toList()), + new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR), + false + ); + } + + /** + * Nanos doesn't use from range, but we don't get the fancy compile into + * filters because of potential loss of precision. + */ + public void testNanosDoesUseFromRange() throws IOException { + aggregationImplementationChoiceTestCase( + aggregableDateFieldType(true, true, DateFormatter.forPattern("yyyy")), + List.of("2017", "2018"), + new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR), + true + ); + } + + public void testFarFutureDoesNotUseFromRange() throws IOException { + aggregationImplementationChoiceTestCase( + aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyyyy")), + List.of("402017", "402018"), + new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR), + false + ); + } + + public void testMissingValueDoesNotUseFromRange() throws IOException { + aggregationImplementationChoiceTestCase( + aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyy")), + List.of("2017", "2018"), + new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR).missing("2020"), + false + ); + } + + private void aggregationImplementationChoiceTestCase( + DateFieldMapper.DateFieldType ft, + List data, + DateHistogramAggregationBuilder builder, + boolean usesFromRange + ) throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + for (String d : data) { + long instant = asLong(d, ft); + indexWriter.addDocument( + List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant), new LongPoint(AGGREGABLE_DATE, instant)) + ); + } + try (IndexReader reader = indexWriter.getReader()) { + SearchContext context = createSearchContext(new IndexSearcher(reader), new MatchAllDocsQuery(), ft); + Aggregator agg = createAggregator(builder, context); + Matcher matcher = instanceOf(DateHistogramAggregator.FromDateRange.class); + if (usesFromRange == false) { + matcher = not(matcher); + } + assertThat(agg, matcher); + agg.preCollection(); + context.searcher().search(context.query(), agg); + InternalDateHistogram result = (InternalDateHistogram) agg.buildTopLevel(); + assertThat(result.getBuckets().stream().map(InternalDateHistogram.Bucket::getKeyAsString).collect(toList()), equalTo(data)); + } + } + } + public void testIllegalInterval() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testSearchCase(new MatchAllDocsQuery(), Collections.emptyList(), diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java index 00def8a016ef8..00ed8e34c7b0f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.search.aggregations.bucket.range; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -37,6 +39,7 @@ import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; @@ -51,14 +54,16 @@ import java.util.function.Consumer; import static java.util.Collections.singleton; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; public class DateRangeAggregatorTests extends AggregatorTestCase { private static final String NUMBER_FIELD_NAME = "number"; private static final String DATE_FIELD_NAME = "date"; - private Instant t1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant(); - private Instant t2 = ZonedDateTime.of(2016, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant(); + private static final Instant T1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant(); + private static final Instant T2 = ZonedDateTime.of(2016, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant(); public void testNoMatchingField() throws IOException { testBothResolutions(new MatchAllDocsQuery(), (iw, resolution) -> { @@ -76,8 +81,18 @@ public void testNoMatchingField() throws IOException { public void testMatchesSortedNumericDocValues() throws IOException { testBothResolutions(new MatchAllDocsQuery(), (iw, resolution) -> { - iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t1)))); - iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t2)))); + iw.addDocument( + List.of( + new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T1)), + new LongPoint(DATE_FIELD_NAME, resolution.convert(T1)) + ) + ); + iw.addDocument( + List.of( + new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T2)), + new LongPoint(DATE_FIELD_NAME, resolution.convert(T2)) + ) + ); }, range -> { List ranges = range.getBuckets(); assertEquals(2, ranges.size()); @@ -89,8 +104,18 @@ public void testMatchesSortedNumericDocValues() throws IOException { public void testMatchesNumericDocValues() throws IOException { testBothResolutions(new MatchAllDocsQuery(), (iw, resolution) -> { - iw.addDocument(singleton(new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t1)))); - iw.addDocument(singleton(new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t2)))); + iw.addDocument( + List.of( + new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T1)), + new LongPoint(DATE_FIELD_NAME, resolution.convert(T1)) + ) + ); + iw.addDocument( + List.of( + new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T2)), + new LongPoint(DATE_FIELD_NAME, resolution.convert(T2)) + ) + ); }, range -> { List ranges = range.getBuckets(); assertEquals(2, ranges.size()); @@ -109,8 +134,8 @@ public void testMissingDateStringWithDateField() throws IOException { .addRange("2015-11-13", "2015-11-14"); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, t1.toEpochMilli()))); - iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, t2.toEpochMilli()))); + iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, T1.toEpochMilli()))); + iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, T2.toEpochMilli()))); // Missing will apply to this document iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7))); }, range -> { @@ -121,6 +146,46 @@ public void testMissingDateStringWithDateField() throws IOException { }, fieldType); } + public void testUnboundedRanges() throws IOException { + testCase( + new RangeAggregationBuilder("name").field(DATE_FIELD_NAME).addUnboundedTo(5).addUnboundedFrom(5), + new MatchAllDocsQuery(), + iw -> { + iw.addDocument( + List.of(new NumericDocValuesField(DATE_FIELD_NAME, Long.MIN_VALUE), new LongPoint(DATE_FIELD_NAME, Long.MIN_VALUE)) + ); + iw.addDocument(List.of(new NumericDocValuesField(DATE_FIELD_NAME, 7), new LongPoint(DATE_FIELD_NAME, 7))); + iw.addDocument(List.of(new NumericDocValuesField(DATE_FIELD_NAME, 2), new LongPoint(DATE_FIELD_NAME, 2))); + iw.addDocument(List.of(new NumericDocValuesField(DATE_FIELD_NAME, 3), new LongPoint(DATE_FIELD_NAME, 3))); + iw.addDocument( + List.of(new NumericDocValuesField(DATE_FIELD_NAME, Long.MAX_VALUE), new LongPoint(DATE_FIELD_NAME, Long.MAX_VALUE)) + ); + }, + result -> { + InternalRange range = (InternalRange) result; + List ranges = range.getBuckets(); + assertThat(ranges, hasSize(2)); + assertThat(ranges.get(0).getFrom(), equalTo(Double.NEGATIVE_INFINITY)); + assertThat(ranges.get(0).getTo(), equalTo(5d)); + assertThat(ranges.get(0).getDocCount(), equalTo(3L)); + assertThat(ranges.get(1).getFrom(), equalTo(5d)); + assertThat(ranges.get(1).getTo(), equalTo(Double.POSITIVE_INFINITY)); + assertThat(ranges.get(1).getDocCount(), equalTo(2L)); + assertTrue(AggregationInspectionHelper.hasValue(range)); + }, + new DateFieldMapper.DateFieldType( + DATE_FIELD_NAME, + randomBoolean(), + randomBoolean(), + true, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + Resolution.MILLISECONDS, + null, + null + ) + ); + } + public void testNumberFieldDateRanges() throws IOException { DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range") .field(NUMBER_FIELD_NAME) @@ -145,8 +210,8 @@ public void testNumberFieldNumberRanges() throws IOException { = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7))); - iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1))); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7))); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 1), new IntPoint(NUMBER_FIELD_NAME, 1))); }, range -> { List ranges = range.getBuckets(); assertEquals(1, ranges.size()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index dda30646a6ca5..7e00046ad1b61 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.search.aggregations.bucket.range; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -32,9 +34,11 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; @@ -48,6 +52,7 @@ import static java.util.Collections.singleton; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; public class RangeAggregatorTests extends AggregatorTestCase { @@ -70,9 +75,9 @@ public void testNoMatchingField() throws IOException { public void testMatchesSortedNumericDocValues() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7))); - iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 2))); - iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 3))); + iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7))); + iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 2), new IntPoint(NUMBER_FIELD_NAME, 2))); + iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 3), new IntPoint(NUMBER_FIELD_NAME, 3))); }, range -> { List ranges = range.getBuckets(); assertEquals(2, ranges.size()); @@ -84,9 +89,9 @@ public void testMatchesSortedNumericDocValues() throws IOException { public void testMatchesNumericDocValues() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7))); - iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 2))); - iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 3))); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7))); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 2), new IntPoint(NUMBER_FIELD_NAME, 2))); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 3), new IntPoint(NUMBER_FIELD_NAME, 3))); }, range -> { List ranges = range.getBuckets(); assertEquals(2, ranges.size()); @@ -96,8 +101,63 @@ public void testMatchesNumericDocValues() throws IOException { }); } + public void testUnboundedRanges() throws IOException { + testCase( + new RangeAggregationBuilder("name").field(NUMBER_FIELD_NAME).addUnboundedTo(5).addUnboundedFrom(5), + new MatchAllDocsQuery(), + iw -> { + iw.addDocument( + List.of( + new NumericDocValuesField(NUMBER_FIELD_NAME, Integer.MIN_VALUE), + new IntPoint(NUMBER_FIELD_NAME, Integer.MIN_VALUE) + ) + ); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7))); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 2), new IntPoint(NUMBER_FIELD_NAME, 2))); + iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 3), new IntPoint(NUMBER_FIELD_NAME, 3))); + iw.addDocument( + List.of( + new NumericDocValuesField(NUMBER_FIELD_NAME, Integer.MAX_VALUE), + new IntPoint(NUMBER_FIELD_NAME, Integer.MAX_VALUE) + ) + ); + }, + result -> { + InternalRange range = (InternalRange) result; + List ranges = range.getBuckets(); + assertThat(ranges, hasSize(2)); + assertThat(ranges.get(0).getFrom(), equalTo(Double.NEGATIVE_INFINITY)); + assertThat(ranges.get(0).getTo(), equalTo(5d)); + assertThat(ranges.get(0).getDocCount(), equalTo(3L)); + assertThat(ranges.get(1).getFrom(), equalTo(5d)); + assertThat(ranges.get(1).getTo(), equalTo(Double.POSITIVE_INFINITY)); + assertThat(ranges.get(1).getDocCount(), equalTo(2L)); + assertTrue(AggregationInspectionHelper.hasValue(range)); + }, + new NumberFieldMapper.NumberFieldType( + NUMBER_FIELD_NAME, + NumberFieldMapper.NumberType.INTEGER, + randomBoolean(), + randomBoolean(), + true, + false, + null, + null + ) + ); + } + public void testDateFieldMillisecondResolution() throws IOException { - DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME); + DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType( + DATE_FIELD_NAME, + randomBoolean(), + randomBoolean(), + true, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + Resolution.MILLISECONDS, + null, + null + ); long milli1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); long milli2 = ZonedDateTime.of(2016, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); @@ -107,8 +167,8 @@ public void testDateFieldMillisecondResolution() throws IOException { .addRange(milli1 - 1, milli1 + 1); testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { - iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli1))); - iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli2))); + iw.addDocument(List.of(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli1), new LongPoint(DATE_FIELD_NAME, milli1))); + iw.addDocument(List.of(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli2), new LongPoint(DATE_FIELD_NAME, milli2))); }, range -> { List ranges = range.getBuckets(); assertEquals(1, ranges.size()); @@ -168,6 +228,39 @@ public void testMissingDateWithDateField() throws IOException { }, fieldType); } + public void testNotFitIntoDouble() throws IOException { + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType( + NUMBER_FIELD_NAME, + NumberType.LONG, + true, + false, + true, + false, + null, + null + ); + + long start = 2L << 54; // Double stores 53 bits of mantissa, so we aggregate a bunch of bigger values + + RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("range") + .field(NUMBER_FIELD_NAME) + .addRange(start, start + 50) + .addRange(start + 50, start + 100) + .addUnboundedFrom(start + 100); + + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (long l = start; l < start + 150; l++) { + iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, l), new LongPoint(NUMBER_FIELD_NAME, l))); + } + }, range -> { + List ranges = range.getBuckets(); + assertThat(ranges, hasSize(3)); + // If we had a native `double` range aggregator we'd get 50, 50, 50 + assertThat(ranges.stream().mapToLong(InternalRange.Bucket::getDocCount).toArray(), equalTo(new long[] {44, 48, 58})); + assertTrue(AggregationInspectionHelper.hasValue(range)); + }, fieldType); + } + public void testMissingDateWithNumberField() throws IOException { RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("range") .field(NUMBER_FIELD_NAME) @@ -295,11 +388,48 @@ public void testSubAggCollectsFromManyBucketsIfManyRanges() throws IOException { }); } + public void testOverlappingRanges() throws IOException { + RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg"); + aggregationBuilder.field(NUMBER_FIELD_NAME); + aggregationBuilder.addRange(0d, 5d); + aggregationBuilder.addRange(10d, 20d); + aggregationBuilder.addRange(0d, 20d); + aggregationBuilder.missing(100); // Set a missing value to force the "normal" range collection instead of filter-based + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 11))); + iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7))); + iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 2))); + iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 3))); + }, result -> { + InternalRange range = (InternalRange) result; + List ranges = range.getBuckets(); + assertThat(ranges, hasSize(3)); + assertThat(ranges.get(0).getFrom(), equalTo(0d)); + assertThat(ranges.get(0).getTo(), equalTo(5d)); + assertThat(ranges.get(0).getDocCount(), equalTo(2L)); + assertThat(ranges.get(1).getFrom(), equalTo(00d)); + assertThat(ranges.get(1).getTo(), equalTo(20d)); + assertThat(ranges.get(1).getDocCount(), equalTo(4L)); + assertThat(ranges.get(2).getFrom(), equalTo(10d)); + assertThat(ranges.get(2).getTo(), equalTo(20d)); + assertThat(ranges.get(2).getDocCount(), equalTo(1L)); + assertTrue(AggregationInspectionHelper.hasValue(range)); + }, new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER)); + } + private void testCase(Query query, CheckedConsumer buildIndex, Consumer> verify) throws IOException { - MappedFieldType fieldType - = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER); + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType( + NUMBER_FIELD_NAME, + NumberFieldMapper.NumberType.INTEGER, + randomBoolean(), + randomBoolean(), + true, + false, + null, + null + ); RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg"); aggregationBuilder.field(NUMBER_FIELD_NAME); aggregationBuilder.addRange(0d, 5d); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java index d15b90bb3783a..9096e763e6038 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java @@ -20,11 +20,14 @@ package org.elasticsearch.search.aggregations.metrics; import com.carrotsearch.hppc.BitMixer; + import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.hamcrest.CoreMatchers; @@ -32,8 +35,8 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; -import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MAX_PRECISION; -import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MIN_PRECISION; +import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MAX_PRECISION; +import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MIN_PRECISION; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -119,4 +122,14 @@ public long addWithoutBreaking(long bytes) { assertThat(total.get(), CoreMatchers.equalTo(0L)); } + + public void testAllocation() { + int precision = between(MIN_PRECISION, MAX_PRECISION); + long initialBucketCount = between(0, 100); + MockBigArrays.assertFitsIn( + ByteSizeValue.ofBytes(Math.max(256, initialBucketCount * 32)), + bigArrays -> new HyperLogLogPlusPlusSparse(precision, bigArrays, initialBucketCount) + ); + } + } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java index 06211a10a024a..e3863ad5b038c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java @@ -21,17 +21,21 @@ import com.carrotsearch.hppc.BitMixer; import com.carrotsearch.hppc.IntHashSet; + import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.test.ESTestCase; import java.util.concurrent.atomic.AtomicLong; -import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MAX_PRECISION; -import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MIN_PRECISION; +import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MAX_PRECISION; +import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MIN_PRECISION; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; @@ -183,4 +187,12 @@ public void testRetrieveCardinality() { } } + public void testAllocation() { + int precision = between(MIN_PRECISION, MAX_PRECISION); + long initialBucketCount = between(0, 100); + MockBigArrays.assertFitsIn( + ByteSizeValue.ofBytes((initialBucketCount << precision) + initialBucketCount * 4 + PageCacheRecycler.PAGE_SIZE_IN_BYTES * 2), + bigArrays -> new HyperLogLogPlusPlus(precision, bigArrays, initialBucketCount) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java index 9d2f38347b40e..36bff44170b4c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java @@ -73,7 +73,7 @@ public void testDatePrepareRoundingWithQuery() throws IOException { MapperService mapperService = dateMapperService(); Query query = mapperService.fieldType("field") .rangeQuery(min, max, true, true, ShapeRelation.CONTAINS, null, null, createQueryShardContext(mapperService)); - withAggregationContext(mapperService, List.of(), query, context -> { + withAggregationContext(null, mapperService, List.of(), query, context -> { Rounding rounding = mock(Rounding.class); CoreValuesSourceType.DATE.getField(context.buildFieldContext("field"), null, context).roundingPreparer().apply(rounding); verify(rounding).prepare(min, max); @@ -102,7 +102,7 @@ public void testDatePrepareRoundingWithDocAndQuery() throws IOException { MapperService mapperService = dateMapperService(); Query query = mapperService.fieldType("field") .rangeQuery(minQuery, maxQuery, true, true, ShapeRelation.CONTAINS, null, null, createQueryShardContext(mapperService)); - withAggregationContext(mapperService, docsWithDatesBetween(minDocs, maxDocs), query, context -> { + withAggregationContext(null, mapperService, docsWithDatesBetween(minDocs, maxDocs), query, context -> { Rounding rounding = mock(Rounding.class); CoreValuesSourceType.DATE.getField(context.buildFieldContext("field"), null, context).roundingPreparer().apply(rounding); verify(rounding).prepare(min, max); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java index 3ac362764ecec..d1caaacafe077 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -55,12 +55,15 @@ public void testEmptyKeyword() throws Exception { LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); assertFalse(values.advanceExact(0)); + assertTrue(config.alignesWithSearchIndex()); + config = ValuesSourceConfig.resolve(context, null, "field", null, "abc", null, null, CoreValuesSourceType.BYTES); valuesSource = (ValuesSource.Bytes) config.getValuesSource(); values = valuesSource.bytesValues(ctx); assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(new BytesRef("abc"), values.nextValue()); + assertFalse(config.alignesWithSearchIndex()); }); } @@ -72,6 +75,7 @@ public void testUnmappedKeyword() throws Exception { ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.getValuesSource(); assertNotNull(valuesSource); assertFalse(config.hasValues()); + assertFalse(config.alignesWithSearchIndex()); config = ValuesSourceConfig.resolve(context, ValueType.STRING, "field", null, "abc", null, null, CoreValuesSourceType.BYTES); valuesSource = (ValuesSource.Bytes) config.getValuesSource(); @@ -80,6 +84,7 @@ public void testUnmappedKeyword() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(new BytesRef("abc"), values.nextValue()); + assertFalse(config.alignesWithSearchIndex()); }); } @@ -94,6 +99,7 @@ public void testLong() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(42, values.nextValue()); + assertTrue(config.alignesWithSearchIndex()); }); } @@ -106,6 +112,7 @@ public void testEmptyLong() throws Exception { LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertFalse(values.advanceExact(0)); + assertTrue(config.alignesWithSearchIndex()); config = ValuesSourceConfig.resolve(context, null, "field", null, 42, null, null, CoreValuesSourceType.BYTES); valuesSource = (ValuesSource.Numeric) config.getValuesSource(); @@ -113,6 +120,7 @@ public void testEmptyLong() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(42, values.nextValue()); + assertFalse(config.alignesWithSearchIndex()); }); } @@ -124,6 +132,7 @@ public void testUnmappedLong() throws Exception { ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.getValuesSource(); assertNotNull(valuesSource); assertFalse(config.hasValues()); + assertFalse(config.alignesWithSearchIndex()); config = ValuesSourceConfig.resolve(context, ValueType.NUMBER, "field", null, 42, null, null, CoreValuesSourceType.BYTES); valuesSource = (ValuesSource.Numeric) config.getValuesSource(); @@ -132,6 +141,7 @@ public void testUnmappedLong() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(42, values.nextValue()); + assertFalse(config.alignesWithSearchIndex()); }); } @@ -146,6 +156,7 @@ public void testBoolean() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(1, values.nextValue()); + assertTrue(config.alignesWithSearchIndex()); }); } @@ -158,6 +169,7 @@ public void testEmptyBoolean() throws Exception { LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); assertFalse(values.advanceExact(0)); + assertTrue(config.alignesWithSearchIndex()); config = ValuesSourceConfig.resolve(context, null, "field", null, true, null, null, CoreValuesSourceType.BYTES); valuesSource = (ValuesSource.Numeric) config.getValuesSource(); @@ -165,6 +177,7 @@ public void testEmptyBoolean() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(1, values.nextValue()); + assertFalse(config.alignesWithSearchIndex()); }); } @@ -176,6 +189,7 @@ public void testUnmappedBoolean() throws Exception { ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.getValuesSource(); assertNotNull(valuesSource); assertFalse(config.hasValues()); + assertFalse(config.alignesWithSearchIndex()); config = ValuesSourceConfig.resolve(context, ValueType.BOOLEAN, "field", null, true, null, null, CoreValuesSourceType.BYTES); valuesSource = (ValuesSource.Numeric) config.getValuesSource(); @@ -184,6 +198,7 @@ public void testUnmappedBoolean() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(1, values.nextValue()); + assertFalse(config.alignesWithSearchIndex()); }); } @@ -214,6 +229,7 @@ public void testFieldAlias() throws Exception { assertTrue(values.advanceExact(0)); assertEquals(1, values.docValueCount()); assertEquals(new BytesRef("value"), values.nextValue()); + assertTrue(config.alignesWithSearchIndex()); }); } } diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java index d91b81f4b8ee9..67447fe029862 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java @@ -138,18 +138,26 @@ public class BootstrapForTesting { // TODO: cut over all tests to bind to ephemeral ports perms.add(new SocketPermission("localhost:1024-", "listen,resolve")); + boolean inGradle = System.getProperty("tests.gradle") != null; + // read test-framework permissions Map codebases = PolicyUtil.getCodebaseJarMap(JarHell.parseClassPath()); // when testing server, the main elasticsearch code is not yet in a jar, so we need to manually add it addClassCodebase(codebases,"elasticsearch", "org.elasticsearch.plugins.PluginsService"); - if (System.getProperty("tests.gradle") == null) { + if (inGradle == false) { // intellij and eclipse don't package our internal libs, so we need to set the codebases for them manually - addClassCodebase(codebases,"plugin-classloader", "org.elasticsearch.plugins.ExtendedPluginsClassLoader"); + addClassCodebase(codebases,"elasticsearch-plugin-classloader", "org.elasticsearch.plugins.ExtendedPluginsClassLoader"); addClassCodebase(codebases,"elasticsearch-nio", "org.elasticsearch.nio.ChannelFactory"); addClassCodebase(codebases, "elasticsearch-secure-sm", "org.elasticsearch.secure_sm.SecureSM"); addClassCodebase(codebases, "elasticsearch-rest-client", "org.elasticsearch.client.RestClient"); } final Policy testFramework = PolicyUtil.readPolicy(Bootstrap.class.getResource("test-framework.policy"), codebases); + final Policy runnerPolicy; + if (inGradle) { + runnerPolicy = PolicyUtil.readPolicy(Bootstrap.class.getResource("gradle.policy"), codebases); + } else { + runnerPolicy = PolicyUtil.readPolicy(Bootstrap.class.getResource("intellij.policy"), codebases); + } // this mimicks the recursive data path permission added in Security.java Permissions fastPathPermissions = new Permissions(); addDirectoryPath(fastPathPermissions, "java.io.tmpdir-fastpath", javaTmpDir, "read,readlink,write,delete", true); @@ -159,7 +167,8 @@ public class BootstrapForTesting { @Override public boolean implies(ProtectionDomain domain, Permission permission) { // implements union - return esPolicy.implies(domain, permission) || testFramework.implies(domain, permission); + return esPolicy.implies(domain, permission) || testFramework.implies(domain, permission) || + runnerPolicy.implies(domain, permission); } }); System.setSecurityManager(SecureSM.createTestSecureSM()); diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java index 284e6a156f7ff..0bb2f05c20035 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java @@ -21,11 +21,20 @@ import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.SeedUtils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -37,12 +46,50 @@ import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import static org.elasticsearch.test.ESTestCase.assertBusy; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MockBigArrays extends BigArrays { + private static final Logger logger = LogManager.getLogger(MockBigArrays.class); + + /** + * Assert that a function returning a {@link Releasable} runs to completion + * when allocated a breaker with that breaks when it uses more than {@code max} + * bytes and that the function doesn't leak any + * {@linkplain BigArray}s if it is given a breaker that allows fewer bytes. + */ + public static void assertFitsIn(ByteSizeValue max, Function run) { + long maxBytes = 0; + long prevLimit = 0; + while (true) { + ByteSizeValue limit = ByteSizeValue.ofBytes(maxBytes); + MockBigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), limit); + Releasable r = null; + try { + r = run.apply(bigArrays); + } catch (CircuitBreakingException e) { + if (maxBytes >= max.getBytes()) { + throw new AssertionError("required more than " + maxBytes + " bytes"); + } + prevLimit = maxBytes; + maxBytes = Math.min(max.getBytes(), maxBytes + Math.max(1, max.getBytes() / 10)); + continue; + } + Releasables.close(r); + logger.info( + "First successfully built using less than {} and more than {}", + ByteSizeValue.ofBytes(maxBytes), + ByteSizeValue.ofBytes(prevLimit) + ); + return; + } + } /** * Tracking allocations is useful when debugging a leak but shouldn't be enabled by default as this would also be very costly @@ -74,6 +121,11 @@ public static void ensureAllArraysAreReleased() throws Exception { exception.addSuppressed((Throwable) cause); } } + if (TRACK_ALLOCATIONS) { + for (Object allocation : masterCopy.values()) { + exception.addSuppressed((Throwable) allocation); + } + } throw exception; } } @@ -84,6 +136,14 @@ public static void ensureAllArraysAreReleased() throws Exception { private final PageCacheRecycler recycler; private final CircuitBreakerService breakerService; + /** + * Create {@linkplain BigArrays} with a configured limit. + */ + public MockBigArrays(PageCacheRecycler recycler, ByteSizeValue limit) { + this(recycler, mock(CircuitBreakerService.class), true); + when(breakerService.getBreaker(CircuitBreaker.REQUEST)).thenReturn(new LimitedBreaker(CircuitBreaker.REQUEST, limit)); + } + public MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService) { this(recycler, breakerService, false); } @@ -263,9 +323,10 @@ private abstract static class AbstractArrayWrapper { AbstractArrayWrapper(boolean clearOnResize) { this.clearOnResize = clearOnResize; this.originalRelease = new AtomicReference<>(); - ACQUIRED_ARRAYS.put(this, - TRACK_ALLOCATIONS ? new RuntimeException("Unreleased array from test: " + LuceneTestCase.getTestClass().getName()) - : Boolean.TRUE); + Object marker = TRACK_ALLOCATIONS + ? new RuntimeException("Array allocated from test: " + LuceneTestCase.getTestClass().getName()) + : true; + ACQUIRED_ARRAYS.put(this, marker); } protected abstract BigArray getDelegate(); @@ -567,4 +628,27 @@ public Collection getChildResources() { } } + private static class LimitedBreaker extends NoopCircuitBreaker { + private final AtomicLong used = new AtomicLong(); + private final ByteSizeValue max; + + LimitedBreaker(String name, ByteSizeValue max) { + super(name); + this.max = max; + } + + @Override + public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException { + long total = used.addAndGet(bytes); + if (total > max.getBytes()) { + throw new CircuitBreakingException("test error", bytes, max.getBytes(), Durability.TRANSIENT); + } + return total; + } + + @Override + public long addWithoutBreaking(long bytes) { + return used.addAndGet(bytes); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index aa9c0c42c1eb2..ca2f9a36b0044 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -28,6 +28,8 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -40,14 +42,19 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.Weight; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.replication.ReplicationResponse; @@ -55,6 +62,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.common.CheckedBiFunction; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; @@ -94,6 +102,7 @@ import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.SearcherHelper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; @@ -110,6 +119,7 @@ import org.junit.Before; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.Charset; import java.nio.file.Path; import java.util.ArrayList; @@ -1184,6 +1194,14 @@ public static MapperService createMapperService() throws IOException { return mapperService; } + public static DocumentMapper docMapper() { + try { + return createMapperService().documentMapper(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + /** * Exposes a translog associated with the given engine for testing purpose. */ @@ -1259,4 +1277,78 @@ public static long getInFlightDocCount(Engine engine) { public static void assertNoInFlightDocuments(Engine engine) throws Exception { assertBusy(() -> assertThat(getInFlightDocCount(engine), equalTo(0L))); } + + public static final class MatchingDirectoryReader extends FilterDirectoryReader { + private final Query query; + + public MatchingDirectoryReader(DirectoryReader in, Query query) throws IOException { + super(in, new SubReaderWrapper() { + @Override + public LeafReader wrap(LeafReader leaf) { + try { + final IndexSearcher searcher = new IndexSearcher(leaf); + final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f); + final Scorer scorer = weight.scorer(leaf.getContext()); + final DocIdSetIterator iterator = scorer != null ? scorer.iterator() : null; + final FixedBitSet liveDocs = new FixedBitSet(leaf.maxDoc()); + if (iterator != null) { + for (int docId = iterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) { + if (leaf.getLiveDocs() == null || leaf.getLiveDocs().get(docId)) { + liveDocs.set(docId); + } + } + } + return new FilterLeafReader(leaf) { + @Override + public Bits getLiveDocs() { + return liveDocs; + } + + @Override + public CacheHelper getCoreCacheHelper() { + return leaf.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return null; // modify liveDocs + } + }; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + }); + this.query = query; + } + + @Override + protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { + return new MatchingDirectoryReader(in, query); + } + + @Override + public CacheHelper getReaderCacheHelper() { + // TODO: We should not return the ReaderCacheHelper if we modify the liveDocs, + // but some caching components (e.g., global ordinals) require this cache key. + return in.getReaderCacheHelper(); + } + } + + public static CheckedFunction randomReaderWrapper() { + if (randomBoolean()) { + return reader -> reader; + } else { + return reader -> new MatchingDirectoryReader(reader, new MatchAllDocsQuery()); + } + } + + public static Function randomSearcherWrapper() { + if (randomBoolean()) { + return Function.identity(); + } else { + final CheckedFunction readerWrapper = randomReaderWrapper(); + return searcher -> SearcherHelper.wrapSearcher(searcher, readerWrapper); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index d8f4d15c9770f..d3b236df26173 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -281,7 +281,12 @@ protected final XContentBuilder fieldMapping(CheckedConsumer docs, CheckedConsumer test ) throws IOException { - withAggregationContext(mapperService, docs, null, test); + withAggregationContext(null, mapperService, docs, null, test); } protected final void withAggregationContext( + ValuesSourceRegistry valuesSourceRegistry, MapperService mapperService, List docs, Query query, @@ -381,7 +387,7 @@ protected final void withAggregationContext( writer.addDocuments(mapperService.documentMapper().parse(doc).docs()); } }, - reader -> test.accept(aggregationContext(mapperService, new IndexSearcher(reader), query)) + reader -> test.accept(aggregationContext(valuesSourceRegistry, mapperService, new IndexSearcher(reader), query)) ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index c9fe7ac9a3199..d677891743cfa 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -367,6 +367,9 @@ protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMe storeProvider = is -> createStore(is, shardPath); } final Store store = storeProvider.apply(indexSettings); + if (indexReaderWrapper == null && randomBoolean()) { + indexReaderWrapper = EngineTestCase.randomReaderWrapper(); + } boolean success = false; try { IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null); diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/SearcherHelper.java b/test/framework/src/main/java/org/elasticsearch/index/shard/SearcherHelper.java new file mode 100644 index 0000000000000..dd5a769c8d999 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/SearcherHelper.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.index.engine.Engine; + +import java.io.IOException; +import java.io.UncheckedIOException; + +public class SearcherHelper { + + public static Engine.Searcher wrapSearcher(Engine.Searcher engineSearcher, + CheckedFunction readerWrapper) { + try { + return IndexShard.wrapSearcher(engineSearcher, readerWrapper); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 0f4afb09e130d..f42c0d2c2881c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -646,7 +646,8 @@ private void wipeCluster() throws Exception { protected static void wipeAllIndices() throws IOException { boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0); try { - final Request deleteRequest = new Request("DELETE", "*"); + //remove all indices except ilm history which can pop up after deleting all data streams but shouldn't interfere + final Request deleteRequest = new Request("DELETE", "*,-.ds-ilm-history-*"); deleteRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); RequestOptions allowSystemIndexAccessWarningOptions = RequestOptions.DEFAULT.toBuilder() .setWarningsHandler(warnings -> { diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc index 224727824cdd0..19eefe18b6f04 100644 --- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc +++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc @@ -889,12 +889,12 @@ in {es}. See <> The realm is designed with the assumption that there needs to be a privileged entity acting as an authentication proxy. In this case, the custom web application is the -authentication proxy handling the authentication of end users ( more correctly, -"delegating" the authentication to the SAML Identity Provider ). The SAML related +authentication proxy handling the authentication of end users (more correctly, +"delegating" the authentication to the SAML Identity Provider). The SAML related APIs require authentication and the necessary authorization level for the authenticated user. For this reason, you must create a Service Account user and assign it a role that gives it the `manage_saml` cluster privilege. The use of the `manage_token` -cluster privilege will be necessary after the authentication takes place, so that the +cluster privilege will be necessary after the authentication takes place, so that the service account user can maintain access in order refresh access tokens on behalf of the authenticated users or to subsequently log them out. diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java index 2878f41da6efe..fd3ff11b5bef0 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java @@ -6,23 +6,9 @@ package org.elasticsearch.xpack.analytics.rate; -import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import java.util.function.Function; - import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -49,6 +35,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; @@ -58,6 +45,22 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + +import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; + public class RateAggregatorTests extends AggregatorTestCase { /** @@ -330,16 +333,36 @@ public void testKeywordSandwich() throws IOException { testCase(dateHistogramAggregationBuilder, new MatchAllDocsQuery(), iw -> { iw.addDocument( - doc("2010-03-11T01:07:45", new NumericDocValuesField("val", 1), new SortedSetDocValuesField("term", new BytesRef("a"))) + doc( + "2010-03-11T01:07:45", + new NumericDocValuesField("val", 1), + new IntPoint("val", 1), + new SortedSetDocValuesField("term", new BytesRef("a")) + ) ); iw.addDocument( - doc("2010-03-12T01:07:45", new NumericDocValuesField("val", 2), new SortedSetDocValuesField("term", new BytesRef("a"))) + doc( + "2010-03-12T01:07:45", + new NumericDocValuesField("val", 2), + new IntPoint("val", 2), + new SortedSetDocValuesField("term", new BytesRef("a")) + ) ); iw.addDocument( - doc("2010-04-01T03:43:34", new NumericDocValuesField("val", 3), new SortedSetDocValuesField("term", new BytesRef("a"))) + doc( + "2010-04-01T03:43:34", + new NumericDocValuesField("val", 3), + new IntPoint("val", 3), + new SortedSetDocValuesField("term", new BytesRef("a")) + ) ); iw.addDocument( - doc("2010-04-27T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b"))) + doc( + "2010-04-27T03:43:34", + new NumericDocValuesField("val", 4), + new IntPoint("val", 4), + new SortedSetDocValuesField("term", new BytesRef("b")) + ) ); }, (Consumer) dh -> { assertThat(dh.getBuckets(), hasSize(2)); @@ -354,6 +377,80 @@ public void testKeywordSandwich() throws IOException { }, dateType, numType, keywordType); } + public void testKeywordSandwichWithSorting() throws IOException { + MappedFieldType numType = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); + MappedFieldType dateType = dateFieldType(DATE_FIELD); + MappedFieldType keywordType = new KeywordFieldMapper.KeywordFieldType("term"); + RateAggregationBuilder rateAggregationBuilder = new RateAggregationBuilder("my_rate").rateUnit("week").field("val"); + boolean useSum = randomBoolean(); + if (useSum) { + if (randomBoolean()) { + rateAggregationBuilder.rateMode("sum"); + } + } else { + rateAggregationBuilder.rateMode("value_count"); + } + TermsAggregationBuilder termsAggregationBuilder = new TermsAggregationBuilder("my_term").field("term") + .order(BucketOrder.aggregation("my_rate", false)) + .subAggregation(rateAggregationBuilder); + DateHistogramAggregationBuilder dateHistogramAggregationBuilder = new DateHistogramAggregationBuilder("my_date").field(DATE_FIELD) + .calendarInterval(new DateHistogramInterval("week")) + .subAggregation(termsAggregationBuilder); + + testCase(dateHistogramAggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument( + doc("2020-11-02T01:07:45", new NumericDocValuesField("val", 1), new SortedSetDocValuesField("term", new BytesRef("a"))) + ); + iw.addDocument( + doc("2020-11-03T01:07:45", new NumericDocValuesField("val", 2), new SortedSetDocValuesField("term", new BytesRef("a"))) + ); + iw.addDocument( + doc("2020-11-04T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b"))) + ); + iw.addDocument( + doc("2020-11-09T03:43:34", new NumericDocValuesField("val", 30), new SortedSetDocValuesField("term", new BytesRef("a"))) + ); + iw.addDocument( + doc("2020-11-10T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b"))) + ); + iw.addDocument( + doc("2020-11-11T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b"))) + ); + }, (Consumer) dh -> { + assertThat(dh.getBuckets(), hasSize(2)); + if (useSum) { + StringTerms st1 = (StringTerms) dh.getBuckets().get(0).getAggregations().asList().get(0); + assertThat(st1.getBuckets(), hasSize(2)); + assertThat(st1.getBuckets().get(0).getKeyAsString(), equalTo("b")); + assertThat(((InternalRate) st1.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(4.0, 0.000001)); + assertThat(st1.getBuckets().get(1).getKeyAsString(), equalTo("a")); + assertThat(((InternalRate) st1.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(3.0, 0.000001)); + + StringTerms st2 = (StringTerms) dh.getBuckets().get(1).getAggregations().asList().get(0); + assertThat(st2.getBuckets(), hasSize(2)); + assertThat(st2.getBuckets().get(0).getKeyAsString(), equalTo("a")); + assertThat(((InternalRate) st2.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(30.0, 0.000001)); + assertThat(st2.getBuckets().get(1).getKeyAsString(), equalTo("b")); + assertThat(((InternalRate) st2.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(8.0, 0.000001)); + } else { + StringTerms st1 = (StringTerms) dh.getBuckets().get(0).getAggregations().asList().get(0); + assertThat(st1.getBuckets(), hasSize(2)); + assertThat(st1.getBuckets().get(0).getKeyAsString(), equalTo("a")); + assertThat(((InternalRate) st1.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(2.0, 0.000001)); + assertThat(st1.getBuckets().get(1).getKeyAsString(), equalTo("b")); + assertThat(((InternalRate) st1.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(1.0, 0.000001)); + + StringTerms st2 = (StringTerms) dh.getBuckets().get(1).getAggregations().asList().get(0); + assertThat(st2.getBuckets(), hasSize(2)); + assertThat(st2.getBuckets().get(0).getKeyAsString(), equalTo("b")); + assertThat(((InternalRate) st2.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(2.0, 0.000001)); + assertThat(st2.getBuckets().get(1).getKeyAsString(), equalTo("a")); + assertThat(((InternalRate) st2.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(1.0, 0.000001)); + + } + }, dateType, numType, keywordType); + } + public void testScriptMonthToDay() throws IOException { testCase( new MatchAllDocsQuery(), @@ -606,6 +703,7 @@ private Iterable doc(String date, IndexableField... fields) { List indexableFields = new ArrayList<>(); long instant = dateFieldType(DATE_FIELD).parse(date); indexableFields.add(new SortedNumericDocValuesField(DATE_FIELD, instant)); + indexableFields.add(new LongPoint(DATE_FIELD, instant)); indexableFields.addAll(Arrays.asList(fields)); return indexableFields; } diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle index 5f3f810fa45bf..97f7ec4d789e3 100644 --- a/x-pack/plugin/async-search/qa/rest/build.gradle +++ b/x-pack/plugin/async-search/qa/rest/build.gradle @@ -2,6 +2,7 @@ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { + name 'test-deprecated-query' description 'Deprecated query plugin' classname 'org.elasticsearch.query.DeprecatedQueryPlugin' } diff --git a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml index 2abf50bf94222..624bbcb99236f 100644 --- a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml +++ b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml @@ -31,3 +31,68 @@ catch: /autoscaling policy with name \[does_not_exist\] does not exist/ autoscaling.delete_autoscaling_policy: name: does_not_exist + +--- +"Test delete all non-existing autoscaling policies": + - do: + autoscaling.delete_autoscaling_policy: + name: "*" + +--- +"Test delete all existing autoscaling policies": + - do: + autoscaling.put_autoscaling_policy: + name: my_autoscaling_policy_1 + body: + roles: [] + + - do: + autoscaling.put_autoscaling_policy: + name: my_autoscaling_policy_2 + body: + roles: [] + + - do: + autoscaling.delete_autoscaling_policy: + name: "*" + + - do: + catch: missing + autoscaling.get_autoscaling_policy: + name: my_autoscaling_policy_1 + + - do: + catch: missing + autoscaling.get_autoscaling_policy: + name: my_autoscaling_policy_2 + +--- +"Test delete autoscaling policies by wildcard": + - do: + autoscaling.put_autoscaling_policy: + name: my_autoscaling_policy_delete + body: + roles: [] + + - do: + autoscaling.put_autoscaling_policy: + name: my_autoscaling_policy_keep + body: + roles: [] + + - do: + autoscaling.delete_autoscaling_policy: + name: "my_autoscaling_policy_delete*" + + - do: + catch: missing + autoscaling.get_autoscaling_policy: + name: my_autoscaling_policy_delete + + - do: + autoscaling.get_autoscaling_policy: + name: my_autoscaling_policy_keep + + - do: + autoscaling.delete_autoscaling_policy: + name: my_autoscaling_policy_keep diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java index 3baadd338684a..94ade2d5fd05f 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java @@ -30,7 +30,8 @@ public void testDeletePolicy() { ); assertAcked(client().execute(PutAutoscalingPolicyAction.INSTANCE, putRequest).actionGet()); // we trust that the policy is in the cluster state since we have tests for putting policies - final DeleteAutoscalingPolicyAction.Request deleteRequest = new DeleteAutoscalingPolicyAction.Request(policy.name()); + String deleteName = randomFrom("*", policy.name(), policy.name().substring(0, between(0, policy.name().length())) + "*"); + final DeleteAutoscalingPolicyAction.Request deleteRequest = new DeleteAutoscalingPolicyAction.Request(deleteName); assertAcked(client().execute(DeleteAutoscalingPolicyAction.INSTANCE, deleteRequest).actionGet()); // now verify that the policy is not in the cluster state final ClusterState state = client().admin().cluster().prepareState().get().getState(); @@ -56,4 +57,9 @@ public void testDeleteNonExistentPolicy() { assertThat(e.getMessage(), containsString("autoscaling policy with name [" + name + "] does not exist")); } + public void testDeleteNonExistentPolicyByWildcard() { + final String name = randomFrom("*", randomAlphaOfLength(8) + "*"); + final DeleteAutoscalingPolicyAction.Request deleteRequest = new DeleteAutoscalingPolicyAction.Request(name); + assertAcked(client().execute(DeleteAutoscalingPolicyAction.INSTANCE, deleteRequest).actionGet()); + } } diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java index 6a785398bbc2a..8b4bfa8760820 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.autoscaling.AutoscalingIntegTestCase; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; @@ -15,7 +16,10 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.mutateAutoscalingDeciders; +import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomAutoscalingDeciders; import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomAutoscalingPolicy; +import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomRoles; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.sameInstance; @@ -57,6 +61,18 @@ public void testNoOpPolicy() { ); } + public void testPutPolicyIllegalName() { + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> putAutoscalingPolicy(new AutoscalingPolicy(randomAlphaOfLength(8) + "*", randomRoles(), randomAutoscalingDeciders())) + ); + + assertThat( + exception.getMessage(), + containsString("name must not contain the following characters " + Strings.INVALID_FILENAME_CHARS) + ); + } + private AutoscalingPolicy putRandomAutoscalingPolicy() { final AutoscalingPolicy policy = randomAutoscalingPolicy(); putAutoscalingPolicy(policy); diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java index 4d6970d89e980..951365e1a541a 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java @@ -6,8 +6,6 @@ package org.elasticsearch.xpack.autoscaling; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.Build; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -45,12 +43,12 @@ import org.elasticsearch.xpack.autoscaling.action.TransportGetAutoscalingCapacityAction; import org.elasticsearch.xpack.autoscaling.action.TransportGetAutoscalingPolicyAction; import org.elasticsearch.xpack.autoscaling.action.TransportPutAutoscalingPolicyAction; +import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCalculateCapacityService; +import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderConfiguration; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult; +import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService; import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderConfiguration; import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderService; -import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderConfiguration; -import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService; -import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCalculateCapacityService; import org.elasticsearch.xpack.autoscaling.rest.RestDeleteAutoscalingPolicyHandler; import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingCapacityHandler; import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingPolicyHandler; @@ -68,7 +66,7 @@ * Container class for autoscaling functionality. */ public class Autoscaling extends Plugin implements ActionPlugin, ExtensiblePlugin, AutoscalingExtension { - private static final Logger logger = LogManager.getLogger(AutoscalingExtension.class); + private static final Boolean AUTOSCALING_FEATURE_FLAG_REGISTERED; static { @@ -91,7 +89,7 @@ public class Autoscaling extends Plugin implements ActionPlugin, ExtensiblePlugi public static final Setting AUTOSCALING_ENABLED_SETTING = Setting.boolSetting( "xpack.autoscaling.enabled", - false, + true, Setting.Property.NodeScope ); diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java index 2c6e86e2c3b0e..07d016c985ebe 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java @@ -8,9 +8,11 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.set.Sets; @@ -136,18 +138,25 @@ public SortedMap deciders() { @Override public ActionRequestValidationException validate() { + ActionRequestValidationException exception = null; if (roles != null) { List errors = roles.stream() .filter(Predicate.not(DiscoveryNode.getPossibleRoleNames()::contains)) .collect(Collectors.toList()); if (errors.isEmpty() == false) { - ActionRequestValidationException exception = new ActionRequestValidationException(); + exception = new ActionRequestValidationException(); exception.addValidationErrors(errors); - return exception; } } - return null; + if (Strings.validFileName(name) == false) { + exception = ValidateActions.addValidationError( + "name must not contain the following characters " + Strings.INVALID_FILENAME_CHARS, + exception + ); + } + + return exception; } @Override diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java index 94e03773a7fc3..e3c772b39870a 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -83,13 +84,20 @@ static ClusterState deleteAutoscalingPolicy(final ClusterState currentState, fin // we will reject the request below when we try to look up the policy by name currentMetadata = AutoscalingMetadata.EMPTY; } - if (currentMetadata.policies().containsKey(name) == false) { + boolean wildcard = Regex.isSimpleMatchPattern(name); + if (wildcard == false && currentMetadata.policies().containsKey(name) == false) { throw new ResourceNotFoundException("autoscaling policy with name [" + name + "] does not exist"); } + final SortedMap newPolicies = new TreeMap<>(currentMetadata.policies()); - final AutoscalingPolicyMetadata policy = newPolicies.remove(name); - assert policy != null : name; - logger.info("deleting autoscaling policy [{}]", name); + if (wildcard) { + newPolicies.keySet().removeIf(key -> Regex.simpleMatch(name, key)); + logger.info("deleting [{}] autoscaling policies", currentMetadata.policies().size() - newPolicies.size()); + } else { + final AutoscalingPolicyMetadata policy = newPolicies.remove(name); + assert policy != null : name; + logger.info("deleting autoscaling policy [{}]", name); + } final AutoscalingMetadata newMetadata = new AutoscalingMetadata(newPolicies); builder.metadata(Metadata.builder(currentState.getMetadata()).putCustom(AutoscalingMetadata.NAME, newMetadata).build()); return builder.build(); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java index 9b4e294875cb8..f0126e321105c 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; @@ -103,6 +104,40 @@ public void testDeletePolicy() { } } + public void testDeletePolicyByWildcard() { + final ClusterState currentState; + { + final ClusterState.Builder builder = ClusterState.builder(new ClusterName(randomAlphaOfLength(8))); + builder.metadata( + Metadata.builder().putCustom(AutoscalingMetadata.NAME, randomAutoscalingMetadataOfPolicyCount(randomIntBetween(1, 8))) + ); + currentState = builder.build(); + } + final AutoscalingMetadata currentMetadata = currentState.metadata().custom(AutoscalingMetadata.NAME); + final String policyName = randomFrom(currentMetadata.policies().keySet()); + final String deleteName = randomFrom(policyName.substring(0, between(0, policyName.length()))) + "*"; + final Logger mockLogger = mock(Logger.class); + final ClusterState state = TransportDeleteAutoscalingPolicyAction.deleteAutoscalingPolicy(currentState, deleteName, mockLogger); + + // ensure the policy is deleted from the cluster state + final AutoscalingMetadata metadata = state.metadata().custom(AutoscalingMetadata.NAME); + assertNotNull(metadata); + assertThat(metadata.policies(), not(hasKey(policyName))); + + verify(mockLogger).info("deleting [{}] autoscaling policies", currentMetadata.policies().size() - metadata.policies().size()); + verifyNoMoreInteractions(mockLogger); + + // ensure that the right policies were preserved + for (final Map.Entry entry : currentMetadata.policies().entrySet()) { + if (Regex.simpleMatch(deleteName, entry.getKey())) { + assertFalse(metadata.policies().containsKey(entry.getKey())); + } else { + assertThat(metadata.policies(), hasKey(entry.getKey())); + assertThat(metadata.policies().get(entry.getKey()).policy(), equalTo(entry.getValue().policy())); + } + } + } + public void testDeleteNonExistentPolicy() { final ClusterState currentState; { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java index f3c7ec44a5249..7d1e0fd101b3d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java @@ -22,7 +22,11 @@ public class RestorableContextClassLoader implements AutoCloseable { private ClassLoader restore; public RestorableContextClassLoader(Class fromClass) throws PrivilegedActionException { - this(Thread.currentThread(), fromClass.getClassLoader()); + this(Thread.currentThread(), getClassLoader(fromClass)); + } + + private static ClassLoader getClassLoader(Class fromClass) throws PrivilegedActionException { + return AccessController.doPrivileged((PrivilegedExceptionAction) fromClass::getClassLoader); } public RestorableContextClassLoader(Thread thread, ClassLoader setClassLoader) throws PrivilegedActionException { diff --git a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy index 964058120f9f4..a0cd3bc741b29 100644 --- a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy @@ -23,11 +23,6 @@ grant codeBase "${codebase.netty-transport}" { permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; }; -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - grant codeBase "${codebase.httpasyncclient}" { // rest client uses system properties which gets the default proxy permission java.net.NetPermission "getProxySelector"; diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index 44ad66d9dabdc..95c3f0133a02f 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -2,15 +2,15 @@ "index_patterns": [ "ilm-history-${xpack.ilm_history.template.version}*" ], + "data_stream": { + "hidden": true + }, "template": { "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0, "index.auto_expand_replicas": "0-1", - "index.lifecycle.name": "ilm-history-ilm-policy", - "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}", - "index.hidden": true, - "index.format": 1 + "index.lifecycle.name": "ilm-history-ilm-policy" }, "mappings": { "dynamic": false, diff --git a/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy index f603bf9ad63ba..16701ab74d8c9 100644 --- a/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy @@ -2,24 +2,3 @@ grant { // needed for multiple server implementations used in tests permission java.net.SocketPermission "*", "accept,connect"; }; - -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; - -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; \ No newline at end of file diff --git a/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy index f603bf9ad63ba..16701ab74d8c9 100644 --- a/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy @@ -2,24 +2,3 @@ grant { // needed for multiple server implementations used in tests permission java.net.SocketPermission "*", "accept,connect"; }; - -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; - -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; \ No newline at end of file diff --git a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy index 65334a8db5760..42baf37c00ba8 100644 --- a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy @@ -4,10 +4,6 @@ grant { // ApacheXMLSecurityInitializer permission java.util.PropertyPermission "org.apache.xml.security.ignoreLineBreaks", "read,write"; - // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader) - permission java.lang.RuntimePermission "getClassLoader"; - permission java.lang.RuntimePermission "setContextClassLoader"; - // needed during initialization of OpenSAML library where xml security algorithms are registered // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220 // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52 @@ -17,13 +13,3 @@ grant { // needed for multiple server implementations used in tests permission java.net.SocketPermission "*", "accept,connect"; }; - -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 3b50caf2fafdb..6895c4e4408cd 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.Nullable; @@ -1210,7 +1211,6 @@ public void testWaitForActiveShardsStep() throws Exception { assertBusy(() -> assertThat(getStepKeyForIndex(client(), originalIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey()))); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/54093") public void testHistoryIsWrittenWithSuccess() throws Exception { createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes"); @@ -1249,7 +1249,6 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { assertBusy(() -> assertHistoryIsPresent(policy, index + "-000002", true, "check-rollover-ready"), 30, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50353") public void testHistoryIsWrittenWithFailure() throws Exception { createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), false); createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); @@ -1267,7 +1266,6 @@ public void testHistoryIsWrittenWithFailure() throws Exception { assertBusy(() -> assertHistoryIsPresent(policy, index + "-1", false, "ERROR"), 30, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53718") public void testHistoryIsWrittenWithDeletion() throws Exception { // Index should be created and then deleted by ILM createIndexWithSettings(client(), index, alias, Settings.builder(), false); @@ -1557,7 +1555,7 @@ private void assertHistoryIsPresent(String policyName, String indexName, boolean } // Finally, check that the history index is in a good state - Step.StepKey stepKey = getStepKeyForIndex(client(), "ilm-history-2-000001"); + Step.StepKey stepKey = getStepKeyForIndex(client(), DataStream.getDefaultBackingIndexName("ilm-history-5", 1)); assertEquals("hot", stepKey.getPhase()); assertEquals(RolloverAction.NAME, stepKey.getAction()); assertEquals(WaitForRolloverReadyStep.NAME, stepKey.getName()); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java deleted file mode 100644 index 808852a9b8411..0000000000000 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.ilm; - -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; -import org.elasticsearch.xpack.core.ilm.LifecycleSettings; -import org.elasticsearch.xpack.core.ilm.OperationMode; -import org.elasticsearch.xpack.core.ilm.Phase; -import org.elasticsearch.xpack.core.ilm.StopILMRequest; -import org.elasticsearch.xpack.core.ilm.action.GetStatusAction; -import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; -import org.elasticsearch.xpack.core.ilm.action.StopILMAction; -import org.elasticsearch.xpack.ilm.history.ILMHistoryStore; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.arrayContaining; -import static org.hamcrest.Matchers.is; - -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1) -public class ILMHistoryTests extends ESIntegTestCase { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal)); - settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false); - settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false); - settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false); - settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false); - settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s"); - settings.put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false); - return settings.build(); - } - - @Override - protected boolean ignoreExternalCluster() { - return true; - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class); - } - - private void putTestPolicy() throws InterruptedException, java.util.concurrent.ExecutionException { - Phase phase = new Phase("hot", TimeValue.ZERO, Collections.emptyMap()); - LifecyclePolicy lifecyclePolicy = new LifecyclePolicy("test", Collections.singletonMap("hot", phase)); - PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy); - assertAcked(client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get()); - } - - public void testIlmHistoryIndexCanRollover() throws Exception { - putTestPolicy(); - Settings settings = Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0).put(LifecycleSettings.LIFECYCLE_NAME, "test").build(); - CreateIndexResponse res = client().admin().indices().prepareCreate("test").setSettings(settings).get(); - assertTrue(res.isAcknowledged()); - - String firstIndex = ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX + "000001"; - String secondIndex = ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX + "000002"; - - assertBusy(() -> { - try { - GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(firstIndex).get(); - assertThat(getIndexResponse.getIndices(), arrayContaining(firstIndex)); - } catch (Exception e) { - fail(e.getMessage()); - } - }); - - //wait for all history items to index to avoid waiting for timeout in ILMHistoryStore beforeBulk - assertBusy(() -> { - try { - SearchResponse search = client().prepareSearch(firstIndex).setQuery(matchQuery("index", firstIndex)).setSize(0).get(); - assertHitCount(search, 9); - } catch (Exception e) { - //assertBusy will stop on first non-assertion error and it can happen when we try to search too early - //instead of failing the whole test change it to assertion error and wait some more time - fail(e.getMessage()); - } - }, 1L, TimeUnit.MINUTES); - - //make sure ILM is stopped so no new items will be queued in ILM history - assertTrue(client().execute(StopILMAction.INSTANCE, new StopILMRequest()).actionGet().isAcknowledged()); - assertBusy(() -> { - GetStatusAction.Response status = client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).actionGet(); - assertThat(status.getMode(), is(OperationMode.STOPPED)); - }); - - RolloverResponse rolloverResponse = client().admin().indices().prepareRolloverIndex(ILMHistoryStore.ILM_HISTORY_ALIAS).get(); - - assertTrue(rolloverResponse.isAcknowledged()); - assertThat(rolloverResponse.getNewIndex(), is(secondIndex)); - - GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(secondIndex).get(); - assertThat(getIndexResponse.getIndices(), arrayContaining(secondIndex)); - } -} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java index c11eb25f4c08c..1a019710b6c08 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java @@ -10,10 +10,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; @@ -22,10 +19,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -33,8 +27,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; @@ -42,14 +34,13 @@ import java.util.Arrays; import java.util.Map; import java.util.Objects; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING; +import static org.elasticsearch.xpack.ilm.history.ILMHistoryTemplateRegistry.ILM_TEMPLATE_NAME; import static org.elasticsearch.xpack.ilm.history.ILMHistoryTemplateRegistry.INDEX_TEMPLATE_VERSION; -import static org.elasticsearch.xpack.ilm.history.ILMHistoryTemplateRegistry.TEMPLATE_ILM_HISTORY; /** * The {@link ILMHistoryStore} handles indexing {@link ILMHistoryItem} documents into the @@ -59,8 +50,7 @@ public class ILMHistoryStore implements Closeable { private static final Logger logger = LogManager.getLogger(ILMHistoryStore.class); - public static final String ILM_HISTORY_INDEX_PREFIX = "ilm-history-" + INDEX_TEMPLATE_VERSION + "-"; - public static final String ILM_HISTORY_ALIAS = "ilm-history-" + INDEX_TEMPLATE_VERSION; + public static final String ILM_HISTORY_DATA_STREAM = "ilm-history-" + INDEX_TEMPLATE_VERSION; private final boolean ilmHistoryEnabled; private final BulkProcessor processor; @@ -75,18 +65,8 @@ public ILMHistoryStore(Settings nodeSettings, Client client, ClusterService clus new BulkProcessor.Listener() { @Override public void beforeBulk(long executionId, BulkRequest request) { - // Prior to actually performing the bulk, we should ensure the index exists, and - // if we were unable to create it or it was in a bad state, we should not - // attempt to index documents. - try { - final CompletableFuture indexCreated = new CompletableFuture<>(); - ensureHistoryIndex(client, clusterService.state(), ActionListener.wrap(indexCreated::complete, - ex -> { - logger.warn("failed to create ILM history store index prior to issuing bulk request", ex); - indexCreated.completeExceptionally(ex); - })); - indexCreated.get(2, TimeUnit.MINUTES); - } catch (Exception e) { + if (clusterService.state().getMetadata().templatesV2().containsKey(ILM_TEMPLATE_NAME) == false) { + ElasticsearchException e = new ElasticsearchException("no ILM history template"); logger.warn(new ParameterizedMessage("unable to index the following ILM history items:\n{}", request.requests().stream() .filter(dwr -> (dwr instanceof IndexRequest)) @@ -150,10 +130,10 @@ public void putAsync(ILMHistoryItem item) { LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), item); return; } - logger.trace("queueing ILM history item for indexing [{}]: [{}]", ILM_HISTORY_ALIAS, item); + logger.trace("queueing ILM history item for indexing [{}]: [{}]", ILM_HISTORY_DATA_STREAM, item); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { item.toXContent(builder, ToXContent.EMPTY_PARAMS); - IndexRequest request = new IndexRequest(ILM_HISTORY_ALIAS).source(builder); + IndexRequest request = new IndexRequest(ILM_HISTORY_DATA_STREAM).source(builder).opType(DocWriteRequest.OpType.CREATE); // TODO: remove the threadpool wrapping when the .add call is non-blocking // (it can currently execute the bulk request occasionally) // see: https://github.com/elastic/elasticsearch/issues/50440 @@ -162,83 +142,12 @@ public void putAsync(ILMHistoryItem item) { processor.add(request); } catch (Exception e) { logger.error(new ParameterizedMessage("failed add ILM history item to queue for index [{}]: [{}]", - ILM_HISTORY_ALIAS, item), e); + ILM_HISTORY_DATA_STREAM, item), e); } }); } catch (IOException exception) { logger.error(new ParameterizedMessage("failed to queue ILM history item in index [{}]: [{}]", - ILM_HISTORY_ALIAS, item), exception); - } - } - - /** - * Checks if the ILM history index exists, and if not, creates it. - * - * @param client The client to use to create the index if needed - * @param state The current cluster state, to determine if the alias exists - * @param listener Called after the index has been created. `onResponse` called with `true` if the index was created, - * `false` if it already existed. - */ - @SuppressWarnings("unchecked") - static void ensureHistoryIndex(Client client, ClusterState state, ActionListener listener) { - final String initialHistoryIndexName = ILM_HISTORY_INDEX_PREFIX + "000001"; - final IndexAbstraction ilmHistory = state.metadata().getIndicesLookup().get(ILM_HISTORY_ALIAS); - final IndexAbstraction initialHistoryIndex = state.metadata().getIndicesLookup().get(initialHistoryIndexName); - - if (ilmHistory == null && initialHistoryIndex == null) { - // No alias or index exists with the expected names, so create the index with appropriate alias - logger.debug("creating ILM history index [{}]", initialHistoryIndexName); - - // Template below should be already defined as real index template but it can be deleted. To avoid race condition with its - // recreation we apply settings and mappings ourselves - byte[] templateBytes = TEMPLATE_ILM_HISTORY.loadBytes(); - Map templateAsMap = XContentHelper.convertToMap(new BytesArray(templateBytes, 0, templateBytes.length), - false, XContentType.JSON).v2(); - templateAsMap = (Map) templateAsMap.get("template"); - - - client.admin().indices().prepareCreate(initialHistoryIndexName) - .setSettings((Map) templateAsMap.get("settings")) - .setMapping((Map) templateAsMap.get("mappings")) - .setWaitForActiveShards(1) - .addAlias(new Alias(ILM_HISTORY_ALIAS).writeIndex(true).isHidden(true)) - .execute(new ActionListener<>() { - @Override - public void onResponse(CreateIndexResponse response) { - listener.onResponse(true); - } - - @Override - public void onFailure(Exception e) { - if (e instanceof ResourceAlreadyExistsException) { - // The index didn't exist before we made the call, there was probably a race - just ignore this - logger.debug("index [{}] was created after checking for its existence, likely due to a concurrent call", - initialHistoryIndexName); - listener.onResponse(false); - } else { - listener.onFailure(e); - } - } - }); - } else if (ilmHistory == null) { - // alias does not exist but initial index does, something is broken - listener.onFailure(new IllegalStateException("ILM history index [" + initialHistoryIndexName + - "] already exists but does not have alias [" + ILM_HISTORY_ALIAS + "]")); - } else if (ilmHistory.getType() == IndexAbstraction.Type.ALIAS) { - if (ilmHistory.getWriteIndex() != null) { - // The alias exists and has a write index, so we're good - listener.onResponse(false); - } else { - // The alias does not have a write index, so we can't index into it - listener.onFailure(new IllegalStateException("ILM history alias [" + ILM_HISTORY_ALIAS + "does not have a write index")); - } - } else if (ilmHistory.getType() != IndexAbstraction.Type.ALIAS) { - // This is not an alias, error out - listener.onFailure(new IllegalStateException("ILM history alias [" + ILM_HISTORY_ALIAS + - "] already exists as " + ilmHistory.getType().getDisplayName())); - } else { - logger.error("unexpected IndexOrAlias for [{}]: [{}]", ILM_HISTORY_ALIAS, ilmHistory); - assert false : ILM_HISTORY_ALIAS + " cannot be both an alias and not an alias simultaneously"; + ILM_HISTORY_DATA_STREAM, item), exception); } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index c8835acbd0b39..a8819d2801dbc 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -30,7 +30,8 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // version 2: convert to hidden index // version 3: templates moved to composable templates // version 4: add `allow_auto_create` setting - public static final int INDEX_TEMPLATE_VERSION = 4; + // version 5: convert to data stream + public static final int INDEX_TEMPLATE_VERSION = 5; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version"; public static final String ILM_TEMPLATE_NAME = "ilm-history"; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java index f22395069d608..e256b0fdcd048 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java @@ -6,14 +6,11 @@ package org.elasticsearch.xpack.ilm.history; -import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -23,36 +20,37 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ilm.LifecycleExecutionState; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.hamcrest.Matchers; import org.junit.After; -import org.junit.Assert; import org.junit.Before; +import java.io.IOException; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING; -import static org.elasticsearch.xpack.ilm.history.ILMHistoryStore.ILM_HISTORY_ALIAS; -import static org.elasticsearch.xpack.ilm.history.ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX; +import static org.elasticsearch.xpack.ilm.history.ILMHistoryStore.ILM_HISTORY_DATA_STREAM; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -68,9 +66,26 @@ public void setup() { threadPool = new TestThreadPool(this.getClass().getName()); client = new VerifyingClient(threadPool); clusterService = ClusterServiceUtils.createClusterService(threadPool); + ILMHistoryTemplateRegistry registry = new ILMHistoryTemplateRegistry(clusterService.getSettings(), clusterService, threadPool, + client, NamedXContentRegistry.EMPTY); + Map templates = + registry.getComposableTemplateConfigs().stream().collect(Collectors.toMap(IndexTemplateConfig::getTemplateName, + this::parseIndexTemplate)); + ClusterState state = clusterService.state(); + ClusterServiceUtils.setState(clusterService, + ClusterState.builder(state).metadata(Metadata.builder(state.metadata()).indexTemplates(templates)).build()); historyStore = new ILMHistoryStore(Settings.EMPTY, client, clusterService, threadPool); } + private ComposableIndexTemplate parseIndexTemplate(IndexTemplateConfig c) { + try { + return ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, c.loadBytes())); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + @After public void setdown() { historyStore.close(); @@ -109,14 +124,11 @@ public void testPut() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> { - if (action instanceof CreateIndexAction && request instanceof CreateIndexRequest) { - return new CreateIndexResponse(true, true, ((CreateIndexRequest) request).index()); - } calledTimes.incrementAndGet(); assertThat(action, instanceOf(BulkAction.class)); assertThat(request, instanceOf(BulkRequest.class)); BulkRequest bulkRequest = (BulkRequest) request; - bulkRequest.requests().forEach(dwr -> assertEquals(ILM_HISTORY_ALIAS, dwr.index())); + bulkRequest.requests().forEach(dwr -> assertEquals(ILM_HISTORY_DATA_STREAM, dwr.index())); assertNotNull(listener); // The content of this BulkResponse doesn't matter, so just make it have the same number of responses @@ -150,7 +162,7 @@ public void testPut() throws Exception { assertThat(request, instanceOf(BulkRequest.class)); BulkRequest bulkRequest = (BulkRequest) request; bulkRequest.requests().forEach(dwr -> { - assertEquals(ILM_HISTORY_ALIAS, dwr.index()); + assertEquals(ILM_HISTORY_DATA_STREAM, dwr.index()); assertThat(dwr, instanceOf(IndexRequest.class)); IndexRequest ir = (IndexRequest) dwr; String indexedDocument = ir.source().utf8ToString(); @@ -173,197 +185,6 @@ public void testPut() throws Exception { } } - public void testHistoryIndexNeedsCreation() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metadata(Metadata.builder()) - .build(); - - client.setVerifier((a, r, l) -> { - assertThat(a, instanceOf(CreateIndexAction.class)); - assertThat(r, instanceOf(CreateIndexRequest.class)); - CreateIndexRequest request = (CreateIndexRequest) r; - assertThat(request.aliases(), Matchers.hasSize(1)); - request.aliases().forEach(alias -> { - assertThat(alias.name(), equalTo(ILM_HISTORY_ALIAS)); - assertTrue(alias.writeIndex()); - }); - return new CreateIndexResponse(true, true, request.index()); - }); - - CountDownLatch latch = new CountDownLatch(1); - ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap( - Assert::assertTrue, - ex -> { - logger.error(ex); - fail("should have called onResponse, not onFailure"); - }), latch)); - - ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testHistoryIndexProperlyExistsAlready() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metadata(Metadata.builder() - .put(IndexMetadata.builder(ILM_HISTORY_INDEX_PREFIX + "000001") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)) - .putAlias(AliasMetadata.builder(ILM_HISTORY_ALIAS) - .writeIndex(true) - .build()))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap( - Assert::assertFalse, - ex -> { - logger.error(ex); - fail("should have called onResponse, not onFailure"); - }), latch)); - - ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testHistoryIndexHasNoWriteIndex() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metadata(Metadata.builder() - .put(IndexMetadata.builder(ILM_HISTORY_INDEX_PREFIX + "000001") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)) - .putAlias(AliasMetadata.builder(ILM_HISTORY_ALIAS) - .build())) - .put(IndexMetadata.builder(randomAlphaOfLength(5)) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)) - .putAlias(AliasMetadata.builder(ILM_HISTORY_ALIAS) - .build()))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap( - indexCreated -> fail("should have called onFailure, not onResponse"), - ex -> { - assertThat(ex, instanceOf(IllegalStateException.class)); - assertThat(ex.getMessage(), Matchers.containsString("ILM history alias [" + ILM_HISTORY_ALIAS + - "does not have a write index")); - }), latch)); - - ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testHistoryIndexNotAlias() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metadata(Metadata.builder() - .put(IndexMetadata.builder(ILM_HISTORY_ALIAS) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap( - indexCreated -> fail("should have called onFailure, not onResponse"), - ex -> { - assertThat(ex, instanceOf(IllegalStateException.class)); - assertThat(ex.getMessage(), Matchers.containsString("ILM history alias [" + ILM_HISTORY_ALIAS + - "] already exists as concrete index")); - }), latch)); - - ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testHistoryIndexCreatedConcurrently() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metadata(Metadata.builder()) - .build(); - - client.setVerifier((a, r, l) -> { - assertThat(a, instanceOf(CreateIndexAction.class)); - assertThat(r, instanceOf(CreateIndexRequest.class)); - CreateIndexRequest request = (CreateIndexRequest) r; - assertThat(request.aliases(), Matchers.hasSize(1)); - request.aliases().forEach(alias -> { - assertThat(alias.name(), equalTo(ILM_HISTORY_ALIAS)); - assertTrue(alias.writeIndex()); - }); - throw new ResourceAlreadyExistsException("that index already exists"); - }); - - CountDownLatch latch = new CountDownLatch(1); - ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap( - Assert::assertFalse, - ex -> { - logger.error(ex); - fail("should have called onResponse, not onFailure"); - }), latch)); - - ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testHistoryAliasDoesntExistButIndexDoes() throws InterruptedException { - final String initialIndex = ILM_HISTORY_INDEX_PREFIX + "000001"; - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metadata(Metadata.builder() - .put(IndexMetadata.builder(initialIndex) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap( - response -> { - logger.error(response); - fail("should have called onFailure, not onResponse"); - }, - ex -> { - assertThat(ex, instanceOf(IllegalStateException.class)); - assertThat(ex.getMessage(), Matchers.containsString("ILM history index [" + initialIndex + - "] already exists but does not have alias [" + ILM_HISTORY_ALIAS + "]")); - }), latch)); - - ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - @SuppressWarnings("unchecked") - private void assertContainsMap(String indexedDocument, Map map) { - map.forEach((k, v) -> { - assertThat(indexedDocument, Matchers.containsString(k)); - if (v instanceof Map) { - assertContainsMap(indexedDocument, (Map) v); - } - if (v instanceof Iterable) { - ((Iterable) v).forEach(elem -> { - assertThat(indexedDocument, Matchers.containsString(elem.toString())); - }); - } else { - assertThat(indexedDocument, Matchers.containsString(v.toString())); - } - }); - } - /** * A client that delegates to a verifying function for action/request/listener */ diff --git a/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy index c54f07cf5cd8d..16701ab74d8c9 100644 --- a/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy @@ -2,24 +2,3 @@ grant { // needed for multiple server implementations used in tests permission java.net.SocketPermission "*", "accept,connect"; }; - -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; - -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java index 88a5183d9f4c9..f08e6f8104042 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java @@ -42,6 +42,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.min; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.hamcrest.Matchers.equalTo; diff --git a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy index 74f27ed286eb6..b23580c25c08f 100644 --- a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy @@ -5,26 +5,3 @@ grant { // needed for Windows named pipes in machine learning permission java.io.FilePermission "\\\\.\\pipe\\*", "read,write"; }; - -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - // Netty sets custom classloader for some of its internal threads - permission java.lang.RuntimePermission "setContextClassLoader"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; - -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; diff --git a/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy index beb104a6b3d09..ef079a5c16e46 100644 --- a/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy @@ -17,23 +17,7 @@ grant { permission java.net.SocketPermission "*", "accept,connect"; }; -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; - grant codeBase "${codebase.elasticsearch-rest-client}" { // rest client uses system properties which gets the default proxy permission java.net.NetPermission "getProxySelector"; }; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; \ No newline at end of file diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java index 3fc545d735d80..56bc692a4767a 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java @@ -15,7 +15,6 @@ import java.util.function.BiConsumer; import java.util.stream.Stream; -import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static java.util.Collections.unmodifiableCollection; import static java.util.Collections.unmodifiableSet; @@ -141,8 +140,8 @@ public String toString() { } @SuppressWarnings("rawtypes") - public static final AttributeMap EMPTY = new AttributeMap<>(); - + private static final AttributeMap EMPTY = new AttributeMap<>(); + @SuppressWarnings("unchecked") public static final AttributeMap emptyAttributeMap() { return EMPTY; @@ -157,23 +156,6 @@ public AttributeMap() { delegate = new LinkedHashMap<>(); } - /** - * Please use the {@link AttributeMap#builder()} instead. - */ - @Deprecated - public AttributeMap(Map attr) { - if (attr.isEmpty()) { - delegate = emptyMap(); - } - else { - delegate = new LinkedHashMap<>(attr.size()); - - for (Entry entry : attr.entrySet()) { - delegate.put(new AttributeWrapper(entry.getKey()), entry.getValue()); - } - } - } - public AttributeMap(Attribute key, E value) { delegate = singletonMap(new AttributeWrapper(key), value); } @@ -377,8 +359,12 @@ public static Builder builder() { return new Builder<>(); } + public static Builder builder(AttributeMap map) { + return new Builder().putAll(map); + } + public static class Builder { - private final AttributeMap map = new AttributeMap<>(); + private AttributeMap map = new AttributeMap<>(); private Builder() {} @@ -393,7 +379,7 @@ public Builder putAll(AttributeMap m) { } public AttributeMap build() { - return new AttributeMap<>(map); + return map; } } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java index 585d3d5da10c9..3c46f42e80450 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java @@ -13,11 +13,9 @@ import java.util.function.Predicate; import java.util.stream.Stream; -import static java.util.Collections.emptyMap; - public class AttributeSet implements Set { - private static final AttributeMap EMPTY_DELEGATE = new AttributeMap<>(emptyMap()); + private static final AttributeMap EMPTY_DELEGATE = AttributeMap.emptyAttributeMap(); public static final AttributeSet EMPTY = new AttributeSet(EMPTY_DELEGATE); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java index 3f716d760654b..c5ce1cea5f54b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java @@ -21,7 +21,6 @@ import java.util.function.Predicate; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; public final class Expressions { @@ -65,7 +64,7 @@ public static List asAttributes(List named public static AttributeMap asAttributeMap(List named) { if (named.isEmpty()) { - return new AttributeMap<>(emptyMap()); + return AttributeMap.emptyAttributeMap(); } AttributeMap map = new AttributeMap<>(); diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java index 41ea7fd44d7ba..9f5d93beb6670 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.xpack.ql.type.DataTypes; import java.util.Collection; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -30,12 +29,12 @@ private static Attribute a(String name) { } private static AttributeMap threeMap() { - Map map = new LinkedHashMap<>(); - map.put(a("one"), "one"); - map.put(a("two"), "two"); - map.put(a("three"), "three"); + AttributeMap.Builder builder = AttributeMap.builder(); + builder.put(a("one"), "one"); + builder.put(a("two"), "two"); + builder.put(a("three"), "three"); - return new AttributeMap<>(map); + return builder.build(); } public void testAttributeMapWithSameAliasesCanResolveAttributes() { @@ -49,19 +48,6 @@ public void testAttributeMapWithSameAliasesCanResolveAttributes() { assertTrue(param1.toAttribute().equals(param2.toAttribute())); assertFalse(param1.toAttribute().semanticEquals(param2.toAttribute())); - Map collectRefs = new LinkedHashMap<>(); - for (Alias a : List.of(param1, param2)) { - collectRefs.put(a.toAttribute(), a.child()); - } - // we can look up the same item by both attributes - assertNotNull(collectRefs.get(param1.toAttribute())); - assertNotNull(collectRefs.get(param2.toAttribute())); - AttributeMap attributeMap = new AttributeMap<>(collectRefs); - - // validate that all Alias can be e - assertTrue(attributeMap.containsKey(param1.toAttribute())); - assertFalse(attributeMap.containsKey(param2.toAttribute())); // results in unknown attribute exception - AttributeMap.Builder mapBuilder = AttributeMap.builder(); for (Alias a : List.of(param1, param2)) { mapBuilder.put(a.toAttribute(), a.child()); @@ -69,7 +55,9 @@ public void testAttributeMapWithSameAliasesCanResolveAttributes() { AttributeMap newAttributeMap = mapBuilder.build(); assertTrue(newAttributeMap.containsKey(param1.toAttribute())); - assertTrue(newAttributeMap.containsKey(param2.toAttribute())); // no more unknown attribute exception + assertTrue(newAttributeMap.get(param1.toAttribute()) == param1.child()); + assertTrue(newAttributeMap.containsKey(param2.toAttribute())); + assertTrue(newAttributeMap.get(param2.toAttribute()) == param2.child()); } private Alias createIntParameterAlias(int index, int value) { @@ -85,13 +73,13 @@ public void testEmptyConstructor() { assertThat(m.isEmpty(), is(true)); } - public void testMapConstructor() { - Map map = new LinkedHashMap<>(); - map.put(a("one"), "one"); - map.put(a("two"), "two"); - map.put(a("three"), "three"); + public void testBuilder() { + AttributeMap.Builder builder = AttributeMap.builder(); + builder.put(a("one"), "one"); + builder.put(a("two"), "two"); + builder.put(a("three"), "three"); - AttributeMap m = new AttributeMap<>(map); + AttributeMap m = builder.build(); assertThat(m.size(), is(3)); assertThat(m.isEmpty(), is(false)); @@ -101,12 +89,7 @@ public void testMapConstructor() { assertThat(m.containsValue("one"), is(true)); assertThat(m.containsValue("on"), is(false)); assertThat(m.attributeNames(), contains("one", "two", "three")); - assertThat(m.values(), contains(map.values().toArray())); - - // defensive copying - map.put(a("four"), "four"); - assertThat(m.size(), is(3)); - assertThat(m.isEmpty(), is(false)); + assertThat(m.values(), contains("one", "two", "three")); } public void testSingleItemConstructor() { @@ -164,12 +147,7 @@ public void testKeySet() { Attribute two = a("two"); Attribute three = a("three"); - Map map = new LinkedHashMap<>(); - map.put(one, "one"); - map.put(two, "two"); - map.put(three, "three"); - - Set keySet = new AttributeMap<>(map).keySet(); + Set keySet = threeMap().keySet(); assertThat(keySet, contains(one, two, three)); // toObject @@ -192,12 +170,7 @@ public void testEntrySet() { Attribute two = a("two"); Attribute three = a("three"); - Map map = new LinkedHashMap<>(); - map.put(one, "one"); - map.put(two, "two"); - map.put(three, "three"); - - Set> set = new AttributeMap<>(map).entrySet(); + Set> set = threeMap().entrySet(); assertThat(set, hasSize(3)); @@ -211,12 +184,9 @@ public void testEntrySet() { assertThat(values, contains("one", "two", "three")); } - public void testForEach() { + public void testCopy() { AttributeMap m = threeMap(); - - Map collect = new LinkedHashMap<>(); - m.forEach(collect::put); - AttributeMap copy = new AttributeMap<>(collect); + AttributeMap copy = AttributeMap.builder().putAll(m).build(); assertThat(m, is(copy)); } diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index 836653c56fbc5..3756275fb2e1c 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -4,10 +4,6 @@ grant { // needed because of problems in unbound LDAP library permission java.util.PropertyPermission "*", "read,write"; - // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader) - permission java.lang.RuntimePermission "getClassLoader"; - permission java.lang.RuntimePermission "setContextClassLoader"; - // needed during initialization of OpenSAML library where xml security algorithms are registered // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220 // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52 @@ -36,24 +32,3 @@ grant { permission java.lang.RuntimePermission "accessUserInformation"; permission java.lang.RuntimePermission "getFileStoreAttributes"; }; - -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; - -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 940afdc458400..1f5564eab3991 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -32,7 +32,7 @@ import org.elasticsearch.xpack.spatial.action.SpatialInfoTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialStatsTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialUsageTransportAction; -import org.elasticsearch.xpack.spatial.aggregations.metrics.GeoShapeCentroidAggregator; +import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractAtomicGeoShapeShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java similarity index 87% rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractAtomicGeoShapeShapeFieldData.java rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java index 32185760bd84c..262b03c2c7d0a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractAtomicGeoShapeShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java @@ -4,11 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.spatial.index.fielddata; +package org.elasticsearch.xpack.spatial.index.fielddata.plain; import org.apache.lucene.util.Accountable; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; +import org.elasticsearch.xpack.spatial.index.fielddata.LeafGeoShapeFieldData; import java.util.Collection; import java.util.Collections; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractLatLonShapeIndexFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractLatLonShapeIndexFieldData.java similarity index 95% rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractLatLonShapeIndexFieldData.java rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractLatLonShapeIndexFieldData.java index 0dc41abfdf8de..df642588ca51c 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractLatLonShapeIndexFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractLatLonShapeIndexFieldData.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.spatial.index.fielddata; +package org.elasticsearch.xpack.spatial.index.fielddata.plain; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; @@ -21,6 +21,8 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.spatial.index.fielddata.IndexGeoShapeFieldData; +import org.elasticsearch.xpack.spatial.index.fielddata.LeafGeoShapeFieldData; public abstract class AbstractLatLonShapeIndexFieldData implements IndexGeoShapeFieldData { protected final String fieldName; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonShapeDVAtomicShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java similarity index 92% rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonShapeDVAtomicShapeFieldData.java rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java index a1eed2faa24b8..97820a2025870 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonShapeDVAtomicShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.spatial.index.fielddata; +package org.elasticsearch.xpack.spatial.index.fielddata.plain; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; @@ -12,6 +12,8 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; +import org.elasticsearch.xpack.spatial.index.fielddata.GeometryDocValueReader; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; import java.io.IOException; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 1f7627968a3e2..5199b9ba9482c 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -29,7 +29,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.xpack.spatial.index.fielddata.AbstractLatLonShapeIndexFieldData; +import org.elasticsearch.xpack.spatial.index.fielddata.plain.AbstractLatLonShapeIndexFieldData; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; import java.util.Arrays; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregator.java similarity index 99% rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregator.java index 8455fa4153f4c..8c6783c2b2394 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregator.java @@ -5,7 +5,7 @@ */ -package org.elasticsearch.xpack.spatial.aggregations.metrics; +package org.elasticsearch.xpack.spatial.search.aggregations.metrics; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.geo.GeoPoint; diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java similarity index 99% rename from x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java rename to x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java index 132af642a62e7..eda3cf5ab4a2b 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.spatial.aggregations.metrics; +package org.elasticsearch.xpack.spatial.search.aggregations.metrics; import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonDocValuesField; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java index 5744d6fbee43d..be4333d55fb3f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java @@ -302,7 +302,7 @@ protected LogicalPlan rule(Project project) { OrderBy ob = (OrderBy) project.child(); // resolve function references (that maybe hiding the target) - final Map collectRefs = new LinkedHashMap<>(); + AttributeMap.Builder collectRefs = AttributeMap.builder(); // collect Attribute sources // only Aliases are interesting since these are the only ones that hide expressions @@ -316,7 +316,7 @@ protected LogicalPlan rule(Project project) { } })); - AttributeMap functions = new AttributeMap<>(collectRefs); + AttributeMap functions = collectRefs.build(); // track the direct parents Map nestedOrders = new LinkedHashMap<>(); @@ -541,14 +541,14 @@ private List combineProjections(List //TODO: this need rewriting when moving functions of NamedExpression // collect aliases in the lower list - Map map = new LinkedHashMap<>(); + AttributeMap.Builder aliasesBuilder = AttributeMap.builder(); for (NamedExpression ne : lower) { if ((ne instanceof Attribute) == false) { - map.put(ne.toAttribute(), ne); + aliasesBuilder.put(ne.toAttribute(), ne); } } - AttributeMap aliases = new AttributeMap<>(map); + AttributeMap aliases = aliasesBuilder.build(); List replaced = new ArrayList<>(); // replace any matching attribute with a lower alias (if there's a match) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java index 55cec8d80a6fd..fb6cb91d03e3e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java @@ -23,9 +23,7 @@ import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import static java.util.Collections.singletonList; @@ -134,7 +132,7 @@ private AttributeSet valuesOutput() { public AttributeMap valuesToLiterals() { AttributeSet outValues = valuesOutput(); - Map valuesMap = new LinkedHashMap<>(); + AttributeMap.Builder valuesMap = AttributeMap.builder(); int index = 0; // for each attribute, associate its value @@ -152,7 +150,7 @@ public AttributeMap valuesToLiterals() { } } - return new AttributeMap<>(valuesMap); + return valuesMap.build(); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java index 9165e1b5d035c..86da3948eaf03 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java @@ -421,20 +421,21 @@ static EsQueryExec fold(AggregateExec a, EsQueryExec exec) { // track aliases defined in the SELECT and used inside GROUP BY // SELECT x AS a ... GROUP BY a - Map aliasMap = new LinkedHashMap<>(); String id = null; + + AttributeMap.Builder aliases = AttributeMap.builder(); for (NamedExpression ne : a.aggregates()) { if (ne instanceof Alias) { - aliasMap.put(ne.toAttribute(), ((Alias) ne).child()); + aliases.put(ne.toAttribute(), ((Alias) ne).child()); } } - if (aliasMap.isEmpty() == false) { - Map newAliases = new LinkedHashMap<>(queryC.aliases()); - newAliases.putAll(aliasMap); - queryC = queryC.withAliases(new AttributeMap<>(newAliases)); + if (aliases.build().isEmpty() == false) { + aliases.putAll(queryC.aliases()); + queryC = queryC.withAliases(aliases.build()); } + // build the group aggregation // NB: any reference in grouping is already "optimized" by its source so there's no need to look for aliases GroupingContext groupingContext = groupBy(a.groupings()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index a2761db533f93..64e4245b557ce 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -432,9 +432,8 @@ public FieldExtraction resolve(Attribute attribute) { // update proc (if needed) if (qContainer.scalarFunctions().size() != scalarFunctions.size()) { - Map procs = new LinkedHashMap<>(qContainer.scalarFunctions()); - procs.put(attr, proc); - qContainer = qContainer.withScalarProcessors(new AttributeMap<>(procs)); + qContainer = qContainer.withScalarProcessors( + AttributeMap.builder(qContainer.scalarFunctions).put(attr, proc).build()); } return new Tuple<>(qContainer, new ComputedRef(proc)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java index 6bee4b338659b..3ef98e6fff235 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java @@ -14,9 +14,7 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.AttributeMap; -import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.ReferenceAttribute; import org.elasticsearch.xpack.ql.querydsl.container.AttributeSort; @@ -32,9 +30,6 @@ import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.sql.querydsl.container.ScoreSort; -import java.util.LinkedHashMap; -import java.util.Map; - import static java.util.Collections.singletonList; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -89,9 +84,7 @@ public void testSortNoneSpecified() { public void testSelectScoreForcesTrackingScore() { Score score = new Score(Source.EMPTY); ReferenceAttribute attr = new ReferenceAttribute(score.source(), "score", score.dataType()); - Map alias = new LinkedHashMap<>(); - alias.put(attr, score); - QueryContainer container = new QueryContainer().withAliases(new AttributeMap<>(alias)).addColumn(attr); + QueryContainer container = new QueryContainer().withAliases(new AttributeMap<>(attr, score)).addColumn(attr); SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10)); assertTrue(sourceBuilder.trackScores()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java index 8886807ad5840..8b7cc542d93b9 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.AttributeMap; -import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.querydsl.query.BoolQuery; import org.elasticsearch.xpack.ql.querydsl.query.MatchAll; @@ -24,8 +23,6 @@ import java.util.AbstractMap.SimpleImmutableEntry; import java.util.Arrays; import java.util.BitSet; -import java.util.LinkedHashMap; -import java.util.Map; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -86,11 +83,8 @@ public void testColumnMaskShouldDuplicateSameAttributes() { Attribute fourth = new FieldAttribute(Source.EMPTY, "fourth", esField); Alias firstAliased = new Alias(Source.EMPTY, "firstAliased", first); - Map aliasesMap = new LinkedHashMap<>(); - aliasesMap.put(firstAliased.toAttribute(), first); - QueryContainer queryContainer = new QueryContainer() - .withAliases(new AttributeMap<>(aliasesMap)) + .withAliases(new AttributeMap<>(firstAliased.toAttribute(), first)) .addColumn(third) .addColumn(first) .addColumn(fourth) diff --git a/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy index 8472a42a64832..d27ded771b86f 100644 --- a/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy @@ -13,24 +13,3 @@ grant { // needed for multiple server implementations used in tests permission java.net.SocketPermission "*", "accept,connect"; }; - -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; - -grant codeBase "${codebase.elasticsearch-rest-client}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; - -grant codeBase "${codebase.httpasyncclient}" { - // rest client uses system properties which gets the default proxy - permission java.net.NetPermission "getProxySelector"; -}; \ No newline at end of file