From 8d1f4565a6b4ef169e926d356519963fdbe78436 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 16 Dec 2024 08:45:06 -0800 Subject: [PATCH 1/4] Update IronBank hardening manifest maintainers (#118175) --- .../docker/src/docker/iron_bank/hardening_manifest.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml index f4364c5008c09..e3bdac51cc5c5 100644 --- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml +++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml @@ -50,9 +50,12 @@ resources: # List of project maintainers maintainers: - - name: "Rory Hunter" - email: "rory.hunter@elastic.co" - username: "rory" + - name: "Mark Vieira" + email: "mark.vieira@elastic.co" + username: "mark-vieira" + - name: "Rene Gröschke" + email: "rene.groschke@elastic.co" + username: "breskeby" - email: "klepal_alexander@bah.com" name: "Alexander Klepal" username: "alexander.klepal" From bf1c0fe0778f1c5af90e6ad1b3da38f444ad114c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Slobodan=20Adamovi=C4=87?= Date: Mon, 16 Dec 2024 19:15:28 +0100 Subject: [PATCH 2/4] Make reserved built-in roles queryable (#117581) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR makes reserved [built-in roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-roles.html) queryable via [Query Role API](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-query-role.html) by indexing them into the `.security` index. Currently, the built-in roles were only available via [Get Role API](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-role.html). The built-in roles are synced into the `.security` index on cluster recovery. The `.security` index will be created (if it's not existing) before built-in roles are synced. In order to avoid concurrent updates, the built-in roles will only be synced by a master node. Once the built-in roles are synced, the information about indexed roles is kept in the cluster state as part of the `.security` index's metadata. The map containing role names and their digests is persisted as part of `queryable_built_in_roles_digest` property: ``` GET /_cluster/state/metadata/.security "queryable_built_in_roles_digest": { "superuser": "lRRmA3kPO1/ztr3ESAlTetOuDjgUC3fKcGS3ZCqM+6k=", ... } ``` Important: The reserved roles stored in the `.security` index are only intended to be used for querying and retrieving. The role resolution and mapping during authentication will remain the same and give a priority to static/file role definitions. This is ensured by the [order in which role providers (built-in, file and native) are invoked](https://github.com/elastic/elasticsearch/blob/71c252c274aa967d5a66f7d081291ac5d87d27a9/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleProviders.java#L77-L81). It’s important to note this because there can be a short period of time where we have a temporary inconsistency between actual built-in role definitions and what is stored in the `.security` index. --- Note: The functionality is temporarily hidden behind the `es.queryable_built_in_roles_enabled` system property. By default, the flag is disabled and will become enabled in a followup PR. The reason for this is to keep this PR as small as possible and to avoid the need to adjust a large number of tests that don't expect `.security` index to exist. Testing: To run and test locally execute `./gradlew run -Dtests.jvm.argline="-Des.queryable_built_in_roles_enabled=true"`. To query all reserved built-in roles execute: ``` POST /_security/_query/role { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } } } ``` --- docs/changelog/117581.yaml | 5 + .../test/rest/ESRestTestCase.java | 2 +- .../security/qa/security-basic/build.gradle | 17 +- .../xpack/security/QueryRoleIT.java | 2 +- .../security/QueryableReservedRolesIT.java | 354 ++++++++++++ .../src/main/java/module-info.java | 6 + .../role/QueryableBuiltInRolesTestPlugin.java | 22 + .../security/src/main/java/module-info.java | 2 + .../xpack/security/Security.java | 23 +- .../xpack/security/SecurityFeatures.java | 8 +- .../action/role/TransportGetRolesAction.java | 12 +- .../security/authz/store/FileRolesStore.java | 9 + .../authz/store/NativeRolesStore.java | 50 +- .../rest/action/role/RestQueryRoleAction.java | 3 + .../support/FeatureNotEnabledException.java | 1 + .../support/QueryableBuiltInRoles.java | 52 ++ .../QueryableBuiltInRolesProviderFactory.java | 23 + .../QueryableBuiltInRolesSynchronizer.java | 532 ++++++++++++++++++ .../support/QueryableBuiltInRolesUtils.java | 101 ++++ .../QueryableReservedRolesProvider.java | 56 ++ .../support/SecurityIndexManager.java | 2 +- .../QueryableBuiltInRolesUtilsTests.java | 296 ++++++++++ .../QueryableReservedRolesProviderTests.java | 31 + 23 files changed, 1585 insertions(+), 24 deletions(-) create mode 100644 docs/changelog/117581.yaml create mode 100644 x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java create mode 100644 x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java create mode 100644 x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRoles.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesProviderFactory.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtils.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProvider.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtilsTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProviderTests.java diff --git a/docs/changelog/117581.yaml b/docs/changelog/117581.yaml new file mode 100644 index 0000000000000..b88017f45e9c9 --- /dev/null +++ b/docs/changelog/117581.yaml @@ -0,0 +1,5 @@ +pr: 117581 +summary: Make reserved built-in roles queryable +area: Authorization +type: enhancement +issues: [] diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 4428afaaeabe5..fa525705a9b39 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1138,7 +1138,7 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE } } - private static boolean ignoreSystemIndexAccessWarnings(List warnings) { + protected static boolean ignoreSystemIndexAccessWarnings(List warnings) { for (String warning : warnings) { if (warning.startsWith("this request accesses system indices:")) { SUITE_LOGGER.warn("Ignoring system index access warning during test cleanup: {}", warning); diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle index 8740354646346..e6caf943dc023 100644 --- a/x-pack/plugin/security/qa/security-basic/build.gradle +++ b/x-pack/plugin/security/qa/security-basic/build.gradle @@ -4,20 +4,31 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +apply plugin: 'elasticsearch.base-internal-es-plugin' apply plugin: 'elasticsearch.internal-java-rest-test' + +esplugin { + name 'queryable-reserved-roles-test' + description 'A test plugin for testing that changes to reserved roles are made queryable' + classname 'org.elasticsearch.xpack.security.role.QueryableBuiltInRolesTestPlugin' + extendedPlugins = ['x-pack-core', 'x-pack-security'] +} dependencies { javaRestTestImplementation(testArtifact(project(xpackModule('security')))) javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + compileOnly project(':x-pack:plugin:core') + compileOnly project(':x-pack:plugin:security') + clusterPlugins project(':x-pack:plugin:security:qa:security-basic') } tasks.named('javaRestTest') { usesDefaultDistribution() } +tasks.named("javadoc").configure { enabled = false } -if (buildParams.inFipsJvm){ +if (buildParams.inFipsJvm) { // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC - tasks.named("javaRestTest").configure{enabled = false } + tasks.named("javaRestTest").configure { enabled = false } } diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java index 1588749b9a331..311510352d805 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java @@ -496,7 +496,7 @@ private RoleDescriptor createRole( ); } - private void assertQuery(String body, int total, Consumer>> roleVerifier) throws IOException { + static void assertQuery(String body, int total, Consumer>> roleVerifier) throws IOException { assertQuery(client(), body, total, roleVerifier); } diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java new file mode 100644 index 0000000000000..7adff21d8df4f --- /dev/null +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryableReservedRolesIT.java @@ -0,0 +1,354 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security; + +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.AnnotationTestOrdering; +import org.elasticsearch.test.AnnotationTestOrdering.Order; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.MutableSettingsProvider; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.local.model.User; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; +import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer; +import org.elasticsearch.xpack.security.support.SecurityMigrations; +import org.junit.BeforeClass; +import org.junit.ClassRule; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.elasticsearch.xpack.security.QueryRoleIT.assertQuery; +import static org.elasticsearch.xpack.security.QueryRoleIT.waitForMigrationCompletion; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.oneOf; + +@TestCaseOrdering(AnnotationTestOrdering.class) +public class QueryableReservedRolesIT extends ESRestTestCase { + + protected static final String REST_USER = "security_test_user"; + private static final SecureString REST_PASSWORD = new SecureString("security-test-password".toCharArray()); + private static final String ADMIN_USER = "admin_user"; + private static final SecureString ADMIN_PASSWORD = new SecureString("admin-password".toCharArray()); + protected static final String READ_SECURITY_USER = "read_security_user"; + private static final SecureString READ_SECURITY_PASSWORD = new SecureString("read-security-password".toCharArray()); + + @BeforeClass + public static void setup() { + new ReservedRolesStore(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + private static MutableSettingsProvider clusterSettings = new MutableSettingsProvider() { + { + put("xpack.license.self_generated.type", "basic"); + put("xpack.security.enabled", "true"); + put("xpack.security.http.ssl.enabled", "false"); + put("xpack.security.transport.ssl.enabled", "false"); + } + }; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .nodes(2) + .settings(clusterSettings) + .rolesFile(Resource.fromClasspath("roles.yml")) + .user(ADMIN_USER, ADMIN_PASSWORD.toString(), User.ROOT_USER_ROLE, true) + .user(REST_USER, REST_PASSWORD.toString(), "security_test_role", false) + .user(READ_SECURITY_USER, READ_SECURITY_PASSWORD.toString(), "read_security_user_role", false) + .systemProperty("es.queryable_built_in_roles_enabled", "true") + .plugin("queryable-reserved-roles-test") + .build(); + + private static Set PREVIOUS_RESERVED_ROLES; + private static Set CONFIGURED_RESERVED_ROLES; + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restAdminSettings() { + String token = basicAuthHeaderValue(ADMIN_USER, ADMIN_PASSWORD); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(REST_USER, REST_PASSWORD); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Order(10) + public void testQueryDeleteOrUpdateReservedRoles() throws Exception { + waitForMigrationCompletion(adminClient(), SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION); + + final String[] allReservedRoles = ReservedRolesStore.names().toArray(new String[0]); + assertQuery(client(), """ + { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 } + """, allReservedRoles.length, roles -> { + assertThat(roles, iterableWithSize(allReservedRoles.length)); + for (var role : roles) { + assertThat((String) role.get("name"), is(oneOf(allReservedRoles))); + } + }); + + final String roleName = randomFrom(allReservedRoles); + assertQuery(client(), String.format(""" + { "query": { "bool": { "must": { "term": { "name": "%s" } } } } } + """, roleName), 1, roles -> { + assertThat(roles, iterableWithSize(1)); + assertThat((String) roles.get(0).get("name"), equalTo(roleName)); + }); + + assertCannotDeleteReservedRoles(); + assertCannotCreateOrUpdateReservedRole(roleName); + } + + @Order(11) + public void testGetReservedRoles() throws Exception { + final String[] allReservedRoles = ReservedRolesStore.names().toArray(new String[0]); + final String roleName = randomFrom(allReservedRoles); + Request request = new Request("GET", "/_security/role/" + roleName); + Response response = adminClient().performRequest(request); + assertOK(response); + var responseMap = responseAsMap(response); + assertThat(responseMap.size(), equalTo(1)); + assertThat(responseMap.containsKey(roleName), is(true)); + } + + @Order(20) + public void testRestartForConfiguringReservedRoles() throws Exception { + configureReservedRoles(List.of("editor", "viewer", "kibana_system", "apm_system", "beats_system", "logstash_system")); + cluster.restart(false); + closeClients(); + } + + @Order(30) + public void testConfiguredReservedRoles() throws Exception { + assert CONFIGURED_RESERVED_ROLES != null; + + // Test query roles API + assertBusy(() -> { + assertQuery(client(), """ + { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 } + """, CONFIGURED_RESERVED_ROLES.size(), roles -> { + assertThat(roles, iterableWithSize(CONFIGURED_RESERVED_ROLES.size())); + for (var role : roles) { + assertThat((String) role.get("name"), is(oneOf(CONFIGURED_RESERVED_ROLES.toArray(new String[0])))); + } + }); + }, 30, TimeUnit.SECONDS); + + // Test get roles API + assertBusy(() -> { + final Response response = adminClient().performRequest(new Request("GET", "/_security/role")); + assertOK(response); + final Map responseMap = responseAsMap(response); + assertThat(responseMap.keySet(), equalTo(CONFIGURED_RESERVED_ROLES)); + }); + } + + @Order(40) + public void testRestartForConfiguringReservedRolesAndClosingIndex() throws Exception { + configureReservedRoles(List.of("editor", "viewer")); + closeSecurityIndex(); + cluster.restart(false); + closeClients(); + } + + @Order(50) + public void testConfiguredReservedRolesAfterClosingAndOpeningIndex() throws Exception { + assert CONFIGURED_RESERVED_ROLES != null; + assert PREVIOUS_RESERVED_ROLES != null; + assertThat(PREVIOUS_RESERVED_ROLES, is(not(equalTo(CONFIGURED_RESERVED_ROLES)))); + + // Test configured roles did not get updated because the security index is closed + assertMetadataContainsBuiltInRoles(PREVIOUS_RESERVED_ROLES); + + // Open the security index + openSecurityIndex(); + + // Test that the roles are now updated after index got opened + assertBusy(() -> { + assertQuery(client(), """ + { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 } + """, CONFIGURED_RESERVED_ROLES.size(), roles -> { + assertThat(roles, iterableWithSize(CONFIGURED_RESERVED_ROLES.size())); + for (var role : roles) { + assertThat((String) role.get("name"), is(oneOf(CONFIGURED_RESERVED_ROLES.toArray(new String[0])))); + } + }); + }, 30, TimeUnit.SECONDS); + + } + + @Order(60) + public void testDeletingAndCreatingSecurityIndexTriggersSynchronization() throws Exception { + deleteSecurityIndex(); + + assertBusy(this::assertSecurityIndexDeleted, 30, TimeUnit.SECONDS); + + // Creating a user will trigger .security index creation + createUser("superman", "superman", "superuser"); + + // Test that the roles are now updated after index got created + assertBusy(() -> { + assertQuery(client(), """ + { "query": { "bool": { "must": { "term": { "metadata._reserved": true } } } }, "size": 100 } + """, CONFIGURED_RESERVED_ROLES.size(), roles -> { + assertThat(roles, iterableWithSize(CONFIGURED_RESERVED_ROLES.size())); + for (var role : roles) { + assertThat((String) role.get("name"), is(oneOf(CONFIGURED_RESERVED_ROLES.toArray(new String[0])))); + } + }); + }, 30, TimeUnit.SECONDS); + } + + private void createUser(String name, String password, String role) throws IOException { + Request request = new Request("PUT", "/_security/user/" + name); + request.setJsonEntity("{ \"password\": \"" + password + "\", \"roles\": [ \"" + role + "\"] }"); + assertOK(adminClient().performRequest(request)); + } + + private void deleteSecurityIndex() throws IOException { + final Request deleteRequest = new Request("DELETE", INTERNAL_SECURITY_MAIN_INDEX_7); + deleteRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(ESRestTestCase::ignoreSystemIndexAccessWarnings)); + final Response response = adminClient().performRequest(deleteRequest); + try (InputStream is = response.getEntity().getContent()) { + assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); + } + } + + private void assertMetadataContainsBuiltInRoles(Set builtInRoles) throws IOException { + final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7); + final Response response = adminClient().performRequest(request); + assertOK(response); + final Map builtInRolesDigests = ObjectPath.createFromResponse(response) + .evaluate("metadata.indices.\\.security-7." + QueryableBuiltInRolesSynchronizer.METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY); + assertThat(builtInRolesDigests.keySet(), equalTo(builtInRoles)); + } + + private void assertSecurityIndexDeleted() throws IOException { + final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7); + final Response response = adminClient().performRequest(request); + assertOK(response); + final Map securityIndexMetadata = ObjectPath.createFromResponse(response) + .evaluate("metadata.indices.\\.security-7"); + assertThat(securityIndexMetadata, is(nullValue())); + } + + private void configureReservedRoles(List reservedRoles) throws Exception { + PREVIOUS_RESERVED_ROLES = CONFIGURED_RESERVED_ROLES; + CONFIGURED_RESERVED_ROLES = new HashSet<>(); + CONFIGURED_RESERVED_ROLES.add("superuser"); // superuser must always be included + CONFIGURED_RESERVED_ROLES.addAll(reservedRoles); + clusterSettings.put("xpack.security.reserved_roles.include", Strings.collectionToCommaDelimitedString(CONFIGURED_RESERVED_ROLES)); + } + + private void closeSecurityIndex() throws Exception { + Request request = new Request("POST", "/" + TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7 + "/_close"); + request.setOptions( + expectWarnings( + "this request accesses system indices: [.security-7], but in a future major version, " + + "direct access to system indices will be prevented by default" + ) + ); + Response response = adminClient().performRequest(request); + assertOK(response); + } + + private void openSecurityIndex() throws Exception { + Request request = new Request("POST", "/" + TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7 + "/_open"); + request.setOptions( + expectWarnings( + "this request accesses system indices: [.security-7], but in a future major version, " + + "direct access to system indices will be prevented by default" + ) + ); + Response response = adminClient().performRequest(request); + assertOK(response); + } + + private void assertCannotDeleteReservedRoles() throws Exception { + { + String roleName = randomFrom(ReservedRolesStore.names()); + Request request = new Request("DELETE", "/_security/role/" + roleName); + var e = expectThrows(ResponseException.class, () -> adminClient().performRequest(request)); + assertThat(e.getMessage(), containsString("role [" + roleName + "] is reserved and cannot be deleted")); + } + { + Request request = new Request("DELETE", "/_security/role/"); + request.setJsonEntity( + """ + { + "names": [%s] + } + """.formatted( + ReservedRolesStore.names().stream().map(name -> "\"" + name + "\"").reduce((a, b) -> a + ", " + b).orElse("") + ) + ); + Response response = adminClient().performRequest(request); + assertOK(response); + String responseAsString = responseAsMap(response).toString(); + for (String roleName : ReservedRolesStore.names()) { + assertThat(responseAsString, containsString("role [" + roleName + "] is reserved and cannot be deleted")); + } + } + } + + private void assertCannotCreateOrUpdateReservedRole(String roleName) throws Exception { + Request request = new Request(randomBoolean() ? "PUT" : "POST", "/_security/role/" + roleName); + request.setJsonEntity(""" + { + "cluster": ["all"], + "indices": [ + { + "names": ["*"], + "privileges": ["all"] + } + ] + } + """); + var e = expectThrows(ResponseException.class, () -> adminClient().performRequest(request)); + assertThat(e.getMessage(), containsString("Role [" + roleName + "] is reserved and may not be used.")); + } + +} diff --git a/x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java b/x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java new file mode 100644 index 0000000000000..00c8e480cfbaf --- /dev/null +++ b/x-pack/plugin/security/qa/security-basic/src/main/java/module-info.java @@ -0,0 +1,6 @@ +module org.elasticsearch.internal.security { + requires org.elasticsearch.base; + requires org.elasticsearch.server; + requires org.elasticsearch.xcore; + requires org.elasticsearch.security; +} diff --git a/x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java b/x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java new file mode 100644 index 0000000000000..ba5538d992cfb --- /dev/null +++ b/x-pack/plugin/security/qa/security-basic/src/main/java/org/elasticsearch/xpack/security/role/QueryableBuiltInRolesTestPlugin.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.role; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; + +import java.util.List; + +public class QueryableBuiltInRolesTestPlugin extends Plugin { + + @Override + public List> getSettings() { + return List.of(ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING); + } +} diff --git a/x-pack/plugin/security/src/main/java/module-info.java b/x-pack/plugin/security/src/main/java/module-info.java index a072b34da7e96..947211559b0c2 100644 --- a/x-pack/plugin/security/src/main/java/module-info.java +++ b/x-pack/plugin/security/src/main/java/module-info.java @@ -70,6 +70,8 @@ exports org.elasticsearch.xpack.security.slowlog to org.elasticsearch.server; exports org.elasticsearch.xpack.security.authc.support to org.elasticsearch.internal.security; exports org.elasticsearch.xpack.security.rest.action.apikey to org.elasticsearch.internal.security; + exports org.elasticsearch.xpack.security.support to org.elasticsearch.internal.security; + exports org.elasticsearch.xpack.security.authz.store to org.elasticsearch.internal.security; provides org.elasticsearch.index.SlowLogFieldProvider with org.elasticsearch.xpack.security.slowlog.SecuritySlowLogFieldProvider; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index ef66392a87260..fd530a338b26c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -411,6 +411,8 @@ import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction; import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry; import org.elasticsearch.xpack.security.support.ExtensionComponents; +import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesProviderFactory; +import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer; import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.support.SecurityMigrationExecutor; @@ -461,6 +463,7 @@ import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE; import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING; import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED; +import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_ENABLED; import static org.elasticsearch.xpack.security.transport.SSLEngineUtils.extractClientCertificates; public class Security extends Plugin @@ -631,7 +634,7 @@ public class Security extends Plugin private final SetOnce reservedRoleNameCheckerFactory = new SetOnce<>(); private final SetOnce fileRoleValidator = new SetOnce<>(); private final SetOnce secondaryAuthActions = new SetOnce<>(); - + private final SetOnce queryableRolesProviderFactory = new SetOnce<>(); private final SetOnce securityMigrationExecutor = new SetOnce<>(); // Node local retry count for migration jobs that's checked only on the master node to make sure @@ -1202,6 +1205,23 @@ Collection createComponents( reservedRoleMappingAction.set(new ReservedRoleMappingAction()); + if (QUERYABLE_BUILT_IN_ROLES_ENABLED) { + if (queryableRolesProviderFactory.get() == null) { + queryableRolesProviderFactory.set(new QueryableBuiltInRolesProviderFactory.Default()); + } + components.add( + new QueryableBuiltInRolesSynchronizer( + clusterService, + featureService, + queryableRolesProviderFactory.get(), + nativeRolesStore, + reservedRolesStore, + fileRolesStore.get(), + threadPool + ) + ); + } + cacheInvalidatorRegistry.validate(); final List reloadableComponents = new ArrayList<>(); @@ -2317,6 +2337,7 @@ public void loadExtensions(ExtensionLoader loader) { loadSingletonExtensionAndSetOnce(loader, grantApiKeyRequestTranslator, RestGrantApiKeyAction.RequestTranslator.class); loadSingletonExtensionAndSetOnce(loader, fileRoleValidator, FileRoleValidator.class); loadSingletonExtensionAndSetOnce(loader, secondaryAuthActions, SecondaryAuthActions.class); + loadSingletonExtensionAndSetOnce(loader, queryableRolesProviderFactory, QueryableBuiltInRolesProviderFactory.class); } private void loadSingletonExtensionAndSetOnce(ExtensionLoader loader, SetOnce setOnce, Class clazz) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java index 53ecafa280715..84749d895a44e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java @@ -12,6 +12,7 @@ import java.util.Set; +import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer.QUERYABLE_BUILT_IN_ROLES_FEATURE; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP; @@ -20,6 +21,11 @@ public class SecurityFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(SECURITY_ROLE_MAPPING_CLEANUP, SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK); + return Set.of( + SECURITY_ROLE_MAPPING_CLEANUP, + SECURITY_ROLES_METADATA_FLATTENED, + SECURITY_MIGRATION_FRAMEWORK, + QUERYABLE_BUILT_IN_ROLES_FEATURE + ); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java index e019f168cf8c0..cdeac51e1f492 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java @@ -20,11 +20,9 @@ import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; -import java.util.List; +import java.util.LinkedHashSet; import java.util.Set; import java.util.stream.Collectors; @@ -51,8 +49,8 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL return; } - final Set rolesToSearchFor = new HashSet<>(); - final List reservedRoles = new ArrayList<>(); + final Set rolesToSearchFor = new LinkedHashSet<>(); + final Set reservedRoles = new LinkedHashSet<>(); if (specificRolesRequested) { for (String role : requestedRoles) { if (ReservedRolesStore.isReserved(role)) { @@ -80,10 +78,10 @@ protected void doExecute(Task task, final GetRolesRequest request, final ActionL } private void getNativeRoles(Set rolesToSearchFor, ActionListener listener) { - getNativeRoles(rolesToSearchFor, new ArrayList<>(), listener); + getNativeRoles(rolesToSearchFor, new LinkedHashSet<>(), listener); } - private void getNativeRoles(Set rolesToSearchFor, List foundRoles, ActionListener listener) { + private void getNativeRoles(Set rolesToSearchFor, Set foundRoles, ActionListener listener) { nativeRolesStore.getRoleDescriptors(rolesToSearchFor, ActionListener.wrap((retrievalResult) -> { if (retrievalResult.isSuccess()) { foundRoles.addAll(retrievalResult.getDescriptors()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java index 7618135c8662f..87378ac0b9f25 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java @@ -44,6 +44,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -173,6 +174,14 @@ public Path getFile() { return file; } + /** + * @return a map of all file role definitions. The returned map is unmodifiable. + */ + public Map getAllRoleDescriptors() { + final Map localPermissions = permissions; + return Collections.unmodifiableMap(localPermissions); + } + // package private for testing Set getAllRoleNames() { return permissions.keySet(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index 23a1fc188e4a0..0a5865ecfe9bf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -63,13 +63,13 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult; import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator; -import org.elasticsearch.xpack.core.security.support.NativeRealmValidationUtil; import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -169,6 +169,10 @@ public NativeRolesStore( this.enabled = settings.getAsBoolean(NATIVE_ROLES_ENABLED, true); } + public boolean isEnabled() { + return enabled; + } + @Override public void accept(Set names, ActionListener listener) { getRoleDescriptors(names, listener); @@ -263,6 +267,10 @@ public boolean isMetadataSearchable() { } public void queryRoleDescriptors(SearchSourceBuilder searchSourceBuilder, ActionListener listener) { + if (enabled == false) { + listener.onResponse(QueryRoleResult.EMPTY); + return; + } SearchRequest searchRequest = new SearchRequest(new String[] { SECURITY_MAIN_ALIAS }, searchSourceBuilder); SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { @@ -345,6 +353,15 @@ public void deleteRoles( final List roleNames, WriteRequest.RefreshPolicy refreshPolicy, final ActionListener listener + ) { + deleteRoles(roleNames, refreshPolicy, true, listener); + } + + public void deleteRoles( + final Collection roleNames, + WriteRequest.RefreshPolicy refreshPolicy, + boolean validateRoleNames, + final ActionListener listener ) { if (enabled == false) { listener.onFailure(new IllegalStateException("Native role management is disabled")); @@ -355,7 +372,7 @@ public void deleteRoles( Map validationErrorByRoleName = new HashMap<>(); for (String roleName : roleNames) { - if (reservedRoleNameChecker.isReserved(roleName)) { + if (validateRoleNames && reservedRoleNameChecker.isReserved(roleName)) { validationErrorByRoleName.put( roleName, new IllegalArgumentException("role [" + roleName + "] is reserved and cannot be deleted") @@ -402,7 +419,7 @@ public void onFailure(Exception e) { } private void bulkResponseAndRefreshRolesCache( - List roleNames, + Collection roleNames, BulkResponse bulkResponse, Map validationErrorByRoleName, ActionListener listener @@ -430,7 +447,7 @@ private void bulkResponseAndRefreshRolesCache( } private void bulkResponseWithOnlyValidationErrors( - List roleNames, + Collection roleNames, Map validationErrorByRoleName, ActionListener listener ) { @@ -542,7 +559,16 @@ public void onFailure(Exception e) { public void putRoles( final WriteRequest.RefreshPolicy refreshPolicy, - final List roles, + final Collection roles, + final ActionListener listener + ) { + putRoles(refreshPolicy, roles, true, listener); + } + + public void putRoles( + final WriteRequest.RefreshPolicy refreshPolicy, + final Collection roles, + boolean validateRoleDescriptors, final ActionListener listener ) { if (enabled == false) { @@ -555,7 +581,7 @@ public void putRoles( for (RoleDescriptor role : roles) { Exception validationException; try { - validationException = validateRoleDescriptor(role); + validationException = validateRoleDescriptors ? validateRoleDescriptor(role) : null; } catch (Exception e) { validationException = e; } @@ -621,8 +647,6 @@ private DeleteRequest createRoleDeleteRequest(final String roleName) { // Package private for testing XContentBuilder createRoleXContentBuilder(RoleDescriptor role) throws IOException { - assert NativeRealmValidationUtil.validateRoleName(role.getName(), false) == null - : "Role name was invalid or reserved: " + role.getName(); assert false == role.hasRestriction() : "restriction is not supported for native roles"; XContentBuilder builder = jsonBuilder().startObject(); @@ -671,7 +695,11 @@ public void usageStats(ActionListener> listener) { client.prepareMultiSearch() .add( client.prepareSearch(SECURITY_MAIN_ALIAS) - .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .setQuery( + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true)) + ) .setTrackTotalHits(true) .setSize(0) ) @@ -680,6 +708,7 @@ public void usageStats(ActionListener> listener) { .setQuery( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true)) .must( QueryBuilders.boolQuery() .should(existsQuery("indices.field_security.grant")) @@ -697,6 +726,7 @@ public void usageStats(ActionListener> listener) { .setQuery( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true)) .filter(existsQuery("indices.query")) ) .setTrackTotalHits(true) @@ -708,6 +738,7 @@ public void usageStats(ActionListener> listener) { .setQuery( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true)) .filter(existsQuery("remote_indices")) ) .setTrackTotalHits(true) @@ -718,6 +749,7 @@ public void usageStats(ActionListener> listener) { .setQuery( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .mustNot(QueryBuilders.termQuery("metadata_flattened._reserved", true)) .filter(existsQuery("remote_cluster")) ) .setTrackTotalHits(true) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java index c2dc7166bd3b6..3637159479463 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestQueryRoleAction.java @@ -14,6 +14,8 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.search.searchafter.SearchAfterBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -32,6 +34,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; +@ServerlessScope(Scope.INTERNAL) public final class RestQueryRoleAction extends NativeRoleBaseRestHandler { @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java index 87c23284c5819..8ba3ebad8a851 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FeatureNotEnabledException.java @@ -29,6 +29,7 @@ public enum Feature { } } + @SuppressWarnings("this-escape") public FeatureNotEnabledException(Feature feature, String message, Object... args) { super(message, args); addMetadata(DISABLED_FEATURE_METADATA, feature.featureName); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRoles.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRoles.java new file mode 100644 index 0000000000000..ec38e4951f45c --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRoles.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.util.Collection; +import java.util.Map; + +/** + * A class that holds the built-in roles and their hash digests. + */ +public record QueryableBuiltInRoles(Map rolesDigest, Collection roleDescriptors) { + + /** + * A listener that is notified when the built-in roles change. + */ + public interface Listener { + + /** + * Called when the built-in roles change. + * + * @param roles the new built-in roles. + */ + void onRolesChanged(QueryableBuiltInRoles roles); + + } + + /** + * A provider that provides the built-in roles and can notify subscribed listeners when the built-in roles change. + */ + public interface Provider { + + /** + * @return the built-in roles. + */ + QueryableBuiltInRoles getRoles(); + + /** + * Adds a listener to be notified when the built-in roles change. + * + * @param listener the listener to add. + */ + void addListener(Listener listener); + + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesProviderFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesProviderFactory.java new file mode 100644 index 0000000000000..c29b64836d1a5 --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesProviderFactory.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.security.authz.store.FileRolesStore; + +public interface QueryableBuiltInRolesProviderFactory { + + QueryableBuiltInRoles.Provider createProvider(ReservedRolesStore reservedRolesStore, FileRolesStore fileRolesStore); + + class Default implements QueryableBuiltInRolesProviderFactory { + @Override + public QueryableBuiltInRoles.Provider createProvider(ReservedRolesStore reservedRolesStore, FileRolesStore fileRolesStore) { + return new QueryableReservedRolesProvider(reservedRolesStore); + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java new file mode 100644 index 0000000000000..60163434e212f --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java @@ -0,0 +1,532 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.TransportActions; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.NotMasterException; +import org.elasticsearch.cluster.SimpleBatchedExecutor; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.component.LifecycleListener; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.engine.DocumentMissingException; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.indices.IndexClosedException; +import org.elasticsearch.indices.IndexPrimaryShardNotAllocatedException; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.security.action.role.BulkRolesResponse; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.security.authz.store.FileRolesStore; +import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; + +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; +import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToDelete; +import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToUpsert; +import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; + +/** + * Synchronizes built-in roles to the .security index. + * The .security index is created if it does not exist. + *

+ * The synchronization is executed only on the elected master node + * after the cluster has recovered and roles need to be synced. + * The goal is to reduce the potential for conflicting operations. + * While in most cases, there should be only a single node that’s + * attempting to create/update/delete roles, it’s still possible + * that the master node changes in the middle of the syncing process. + */ +public final class QueryableBuiltInRolesSynchronizer implements ClusterStateListener { + + private static final Logger logger = LogManager.getLogger(QueryableBuiltInRolesSynchronizer.class); + + /** + * This is a temporary feature flag to allow enabling the synchronization of built-in roles to the .security index. + * Initially, it is disabled by default due to the number of tests that need to be adjusted now that .security index + * is created earlier in the cluster lifecycle. + *

+ * Once all tests are adjusted, this flag will be set to enabled by default and later removed altogether. + */ + public static final boolean QUERYABLE_BUILT_IN_ROLES_ENABLED; + static { + final var propertyValue = System.getProperty("es.queryable_built_in_roles_enabled"); + if (propertyValue == null || propertyValue.isEmpty() || "false".equals(propertyValue)) { + QUERYABLE_BUILT_IN_ROLES_ENABLED = false; + } else if ("true".equals(propertyValue)) { + QUERYABLE_BUILT_IN_ROLES_ENABLED = true; + } else { + throw new IllegalStateException( + "system property [es.queryable_built_in_roles_enabled] may only be set to [true] or [false], but was [" + + propertyValue + + "]" + ); + } + } + + public static final NodeFeature QUERYABLE_BUILT_IN_ROLES_FEATURE = new NodeFeature("security.queryable_built_in_roles"); + + /** + * Index metadata key of the digest of built-in roles indexed in the .security index. + *

+ * The value is a map of built-in role names to their digests (calculated by sha256 of the role definition). + */ + public static final String METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY = "queryable_built_in_roles_digest"; + + private static final SimpleBatchedExecutor> MARK_ROLES_AS_SYNCED_TASK_EXECUTOR = + new SimpleBatchedExecutor<>() { + @Override + public Tuple> executeTask(MarkRolesAsSyncedTask task, ClusterState clusterState) { + return task.execute(clusterState); + } + + @Override + public void taskSucceeded(MarkRolesAsSyncedTask task, Map value) { + task.success(value); + } + }; + + private final MasterServiceTaskQueue markRolesAsSyncedTaskQueue; + + private final ClusterService clusterService; + private final FeatureService featureService; + private final QueryableBuiltInRoles.Provider rolesProvider; + private final NativeRolesStore nativeRolesStore; + private final Executor executor; + private final AtomicBoolean synchronizationInProgress = new AtomicBoolean(false); + + private volatile boolean securityIndexDeleted = false; + + /** + * Constructs a new built-in roles synchronizer. + * + * @param clusterService the cluster service to register as a listener + * @param featureService the feature service to check if the cluster has the queryable built-in roles feature + * @param rolesProviderFactory the factory to create the built-in roles provider + * @param nativeRolesStore the native roles store to sync the built-in roles to + * @param reservedRolesStore the reserved roles store to fetch the built-in roles from + * @param fileRolesStore the file roles store to fetch the built-in roles from + * @param threadPool the thread pool + */ + public QueryableBuiltInRolesSynchronizer( + ClusterService clusterService, + FeatureService featureService, + QueryableBuiltInRolesProviderFactory rolesProviderFactory, + NativeRolesStore nativeRolesStore, + ReservedRolesStore reservedRolesStore, + FileRolesStore fileRolesStore, + ThreadPool threadPool + ) { + this.clusterService = clusterService; + this.featureService = featureService; + this.rolesProvider = rolesProviderFactory.createProvider(reservedRolesStore, fileRolesStore); + this.nativeRolesStore = nativeRolesStore; + this.executor = threadPool.generic(); + this.markRolesAsSyncedTaskQueue = clusterService.createTaskQueue( + "mark-built-in-roles-as-synced-task-queue", + Priority.LOW, + MARK_ROLES_AS_SYNCED_TASK_EXECUTOR + ); + this.rolesProvider.addListener(this::builtInRolesChanged); + this.clusterService.addLifecycleListener(new LifecycleListener() { + @Override + public void beforeStop() { + clusterService.removeListener(QueryableBuiltInRolesSynchronizer.this); + } + + @Override + public void beforeStart() { + clusterService.addListener(QueryableBuiltInRolesSynchronizer.this); + } + }); + } + + private void builtInRolesChanged(QueryableBuiltInRoles roles) { + logger.debug("Built-in roles changed, attempting to sync to .security index"); + final ClusterState state = clusterService.state(); + if (shouldSyncBuiltInRoles(state)) { + syncBuiltInRoles(roles); + } + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + final ClusterState state = event.state(); + if (isSecurityIndexDeleted(event)) { + this.securityIndexDeleted = true; + logger.trace("Received security index deletion event, skipping built-in roles synchronization"); + return; + } else if (isSecurityIndexCreatedOrRecovered(event)) { + this.securityIndexDeleted = false; + logger.trace("Security index has been created/recovered, attempting to sync built-in roles"); + } + if (shouldSyncBuiltInRoles(state)) { + final QueryableBuiltInRoles roles = rolesProvider.getRoles(); + syncBuiltInRoles(roles); + } + } + + private void syncBuiltInRoles(final QueryableBuiltInRoles roles) { + if (synchronizationInProgress.compareAndSet(false, true)) { + final Map indexedRolesDigests = readIndexedBuiltInRolesDigests(clusterService.state()); + if (roles.rolesDigest().equals(indexedRolesDigests)) { + logger.debug("Security index already contains the latest built-in roles indexed, skipping synchronization"); + return; + } + executor.execute(() -> doSyncBuiltinRoles(indexedRolesDigests, roles, ActionListener.wrap(v -> { + logger.info("Successfully synced [" + roles.roleDescriptors().size() + "] built-in roles to .security index"); + synchronizationInProgress.set(false); + }, e -> { + handleException(e); + synchronizationInProgress.set(false); + }))); + } + } + + private static void handleException(Exception e) { + if (e instanceof BulkRolesResponseException bulkException) { + final boolean isBulkDeleteFailure = bulkException instanceof BulkDeleteRolesResponseException; + for (final Map.Entry bulkFailure : bulkException.getFailures().entrySet()) { + final String logMessage = Strings.format( + "Failed to [%s] built-in role [%s]", + isBulkDeleteFailure ? "delete" : "create/update", + bulkFailure.getKey() + ); + if (isExpectedFailure(bulkFailure.getValue())) { + logger.info(logMessage, bulkFailure.getValue()); + } else { + logger.warn(logMessage, bulkFailure.getValue()); + } + } + } else if (isExpectedFailure(e)) { + logger.info("Failed to sync built-in roles to .security index", e); + } else { + logger.warn("Failed to sync built-in roles to .security index due to unexpected exception", e); + } + } + + /** + * Some failures are expected and should not be logged as errors. + * These exceptions are either: + * - transient (e.g. connection errors), + * - recoverable (e.g. no longer master, index reallocating or caused by concurrent operations) + * - not recoverable but expected (e.g. index closed). + * + * @param e to check + * @return {@code true} if the exception is expected and should not be logged as an error + */ + private static boolean isExpectedFailure(final Exception e) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + return ExceptionsHelper.isNodeOrShardUnavailableTypeException(cause) + || TransportActions.isShardNotAvailableException(cause) + || cause instanceof IndexClosedException + || cause instanceof IndexPrimaryShardNotAllocatedException + || cause instanceof NotMasterException + || cause instanceof ResourceAlreadyExistsException + || cause instanceof VersionConflictEngineException + || cause instanceof DocumentMissingException + || cause instanceof FailedToMarkBuiltInRolesAsSyncedException; + } + + private boolean shouldSyncBuiltInRoles(final ClusterState state) { + if (false == state.nodes().isLocalNodeElectedMaster()) { + logger.trace("Local node is not the master, skipping built-in roles synchronization"); + return false; + } + if (false == state.clusterRecovered()) { + logger.trace("Cluster state has not recovered yet, skipping built-in roles synchronization"); + return false; + } + if (nativeRolesStore.isEnabled() == false) { + logger.trace("Native roles store is not enabled, skipping built-in roles synchronization"); + return false; + } + if (state.nodes().getDataNodes().isEmpty()) { + logger.trace("No data nodes in the cluster, skipping built-in roles synchronization"); + return false; + } + if (state.nodes().isMixedVersionCluster()) { + // To keep things simple and avoid potential overwrites with an older version of built-in roles, + // we only sync built-in roles if all nodes are on the same version. + logger.trace("Not all nodes are on the same version, skipping built-in roles synchronization"); + return false; + } + if (false == featureService.clusterHasFeature(state, QUERYABLE_BUILT_IN_ROLES_FEATURE)) { + logger.trace("Not all nodes support queryable built-in roles feature, skipping built-in roles synchronization"); + return false; + } + if (securityIndexDeleted) { + logger.trace("Security index is deleted, skipping built-in roles synchronization"); + return false; + } + if (isSecurityIndexClosed(state)) { + logger.trace("Security index is closed, skipping built-in roles synchronization"); + return false; + } + return true; + } + + private void doSyncBuiltinRoles( + final Map indexedRolesDigests, + final QueryableBuiltInRoles roles, + final ActionListener listener + ) { + final Set rolesToUpsert = determineRolesToUpsert(roles, indexedRolesDigests); + final Set rolesToDelete = determineRolesToDelete(roles, indexedRolesDigests); + + assert Sets.intersection(rolesToUpsert.stream().map(RoleDescriptor::getName).collect(toSet()), rolesToDelete).isEmpty() + : "The roles to upsert and delete should not have any common roles"; + + if (rolesToUpsert.isEmpty() && rolesToDelete.isEmpty()) { + logger.debug("No changes to built-in roles to sync to .security index"); + listener.onResponse(null); + return; + } + + indexRoles(rolesToUpsert, listener.delegateFailureAndWrap((l1, indexResponse) -> { + deleteRoles(rolesToDelete, l1.delegateFailureAndWrap((l2, deleteResponse) -> { + markRolesAsSynced(indexedRolesDigests, roles.rolesDigest(), l2); + })); + })); + } + + private void deleteRoles(final Set rolesToDelete, final ActionListener listener) { + if (rolesToDelete.isEmpty()) { + listener.onResponse(null); + return; + } + nativeRolesStore.deleteRoles(rolesToDelete, WriteRequest.RefreshPolicy.IMMEDIATE, false, ActionListener.wrap(deleteResponse -> { + final Map deleteFailure = deleteResponse.getItems() + .stream() + .filter(BulkRolesResponse.Item::isFailed) + .collect(toMap(BulkRolesResponse.Item::getRoleName, BulkRolesResponse.Item::getCause)); + if (deleteFailure.isEmpty()) { + listener.onResponse(null); + } else { + listener.onFailure(new BulkDeleteRolesResponseException(deleteFailure)); + } + }, listener::onFailure)); + } + + private void indexRoles(final Collection rolesToUpsert, final ActionListener listener) { + if (rolesToUpsert.isEmpty()) { + listener.onResponse(null); + return; + } + nativeRolesStore.putRoles(WriteRequest.RefreshPolicy.IMMEDIATE, rolesToUpsert, false, ActionListener.wrap(response -> { + final Map indexFailures = response.getItems() + .stream() + .filter(BulkRolesResponse.Item::isFailed) + .collect(toMap(BulkRolesResponse.Item::getRoleName, BulkRolesResponse.Item::getCause)); + if (indexFailures.isEmpty()) { + listener.onResponse(null); + } else { + listener.onFailure(new BulkIndexRolesResponseException(indexFailures)); + } + }, listener::onFailure)); + } + + private boolean isSecurityIndexDeleted(final ClusterChangedEvent event) { + final IndexMetadata previousSecurityIndexMetadata = resolveSecurityIndexMetadata(event.previousState().metadata()); + final IndexMetadata currentSecurityIndexMetadata = resolveSecurityIndexMetadata(event.state().metadata()); + return previousSecurityIndexMetadata != null && currentSecurityIndexMetadata == null; + } + + private boolean isSecurityIndexCreatedOrRecovered(final ClusterChangedEvent event) { + final IndexMetadata previousSecurityIndexMetadata = resolveSecurityIndexMetadata(event.previousState().metadata()); + final IndexMetadata currentSecurityIndexMetadata = resolveSecurityIndexMetadata(event.state().metadata()); + return previousSecurityIndexMetadata == null && currentSecurityIndexMetadata != null; + } + + private boolean isSecurityIndexClosed(final ClusterState state) { + final IndexMetadata indexMetadata = resolveSecurityIndexMetadata(state.metadata()); + return indexMetadata != null && indexMetadata.getState() == IndexMetadata.State.CLOSE; + } + + /** + * This method marks the built-in roles as synced in the .security index + * by setting the new roles digests in the metadata of the .security index. + *

+ * The marking is done as a compare and swap operation to ensure that the roles + * are marked as synced only when new roles are indexed. The operation is idempotent + * and will succeed if the expected roles digests are equal to the digests in the + * .security index or if they are equal to the new roles digests. + */ + private void markRolesAsSynced( + final Map expectedRolesDigests, + final Map newRolesDigests, + final ActionListener listener + ) { + final IndexMetadata securityIndexMetadata = resolveSecurityIndexMetadata(clusterService.state().metadata()); + if (securityIndexMetadata == null) { + listener.onFailure(new IndexNotFoundException(SECURITY_MAIN_ALIAS)); + return; + } + final Index concreteSecurityIndex = securityIndexMetadata.getIndex(); + markRolesAsSyncedTaskQueue.submitTask( + "mark built-in roles as synced task", + new MarkRolesAsSyncedTask(listener.delegateFailureAndWrap((l, response) -> { + if (newRolesDigests.equals(response) == false) { + logger.debug( + () -> Strings.format( + "Another master node most probably indexed a newer versions of built-in roles in the meantime. " + + "Expected: [%s], Actual: [%s]", + newRolesDigests, + response + ) + ); + l.onFailure( + new FailedToMarkBuiltInRolesAsSyncedException( + "Failed to mark built-in roles as synced. The expected role digests have changed." + ) + ); + } else { + l.onResponse(null); + } + }), concreteSecurityIndex.getName(), expectedRolesDigests, newRolesDigests), + null + ); + } + + private Map readIndexedBuiltInRolesDigests(final ClusterState state) { + final IndexMetadata indexMetadata = resolveSecurityIndexMetadata(state.metadata()); + if (indexMetadata == null) { + return null; + } + return indexMetadata.getCustomData(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY); + } + + private static IndexMetadata resolveSecurityIndexMetadata(final Metadata metadata) { + return SecurityIndexManager.resolveConcreteIndex(SECURITY_MAIN_ALIAS, metadata); + } + + static class MarkRolesAsSyncedTask implements ClusterStateTaskListener { + + private final ActionListener> listener; + private final String concreteSecurityIndexName; + private final Map expectedRoleDigests; + private final Map newRoleDigests; + + MarkRolesAsSyncedTask( + ActionListener> listener, + String concreteSecurityIndexName, + @Nullable Map expectedRoleDigests, + @Nullable Map newRoleDigests + ) { + this.listener = listener; + this.concreteSecurityIndexName = concreteSecurityIndexName; + this.expectedRoleDigests = expectedRoleDigests; + this.newRoleDigests = newRoleDigests; + } + + Tuple> execute(ClusterState state) { + IndexMetadata indexMetadata = state.metadata().index(concreteSecurityIndexName); + if (indexMetadata == null) { + throw new IndexNotFoundException(concreteSecurityIndexName); + } + Map existingRoleDigests = indexMetadata.getCustomData(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY); + if (Objects.equals(expectedRoleDigests, existingRoleDigests)) { + IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexMetadata); + if (newRoleDigests != null) { + indexMetadataBuilder.putCustom(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY, newRoleDigests); + } else { + indexMetadataBuilder.removeCustom(METADATA_QUERYABLE_BUILT_IN_ROLES_DIGEST_KEY); + } + indexMetadataBuilder.version(indexMetadataBuilder.version() + 1); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(state.metadata().indices()); + builder.put(concreteSecurityIndexName, indexMetadataBuilder.build()); + return new Tuple<>( + ClusterState.builder(state).metadata(Metadata.builder(state.metadata()).indices(builder.build()).build()).build(), + newRoleDigests + ); + } else { + // returns existing value when expectation is not met + return new Tuple<>(state, existingRoleDigests); + } + } + + void success(Map value) { + listener.onResponse(value); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + } + + private static class BulkDeleteRolesResponseException extends BulkRolesResponseException { + + BulkDeleteRolesResponseException(Map failures) { + super("Failed to bulk delete built-in roles", failures); + } + + } + + private static class BulkIndexRolesResponseException extends BulkRolesResponseException { + + BulkIndexRolesResponseException(Map failures) { + super("Failed to bulk create/update built-in roles", failures); + } + + } + + private abstract static class BulkRolesResponseException extends RuntimeException { + + private final Map failures; + + BulkRolesResponseException(String message, Map failures) { + super(message); + assert failures != null && failures.isEmpty() == false; + this.failures = failures; + failures.values().forEach(this::addSuppressed); + } + + Map getFailures() { + return failures; + } + + } + + private static class FailedToMarkBuiltInRolesAsSyncedException extends RuntimeException { + + FailedToMarkBuiltInRolesAsSyncedException(String message) { + super(message); + } + + } + +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtils.java new file mode 100644 index 0000000000000..2d2eb345594ed --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtils.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; + +/** + * Utility class which provides helper method for calculating the hash of a role descriptor, + * determining the roles to upsert and the roles to delete. + */ +public final class QueryableBuiltInRolesUtils { + + /** + * Calculates the hash of the given role descriptor by serializing it by calling {@link RoleDescriptor#writeTo(StreamOutput)} method + * and then SHA256 hashing the bytes. + * + * @param roleDescriptor the role descriptor to hash + * @return the base64 encoded SHA256 hash of the role descriptor + */ + public static String calculateHash(final RoleDescriptor roleDescriptor) { + final MessageDigest hash = MessageDigests.sha256(); + try (XContentBuilder jsonBuilder = XContentFactory.jsonBuilder()) { + roleDescriptor.toXContent(jsonBuilder, EMPTY_PARAMS); + final Map flattenMap = Maps.flatten( + XContentHelper.convertToMap(BytesReference.bytes(jsonBuilder), true, XContentType.JSON).v2(), + false, + true + ); + hash.update(flattenMap.toString().getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new IllegalStateException("failed to compute digest for [" + roleDescriptor.getName() + "] role", e); + } + // HEX vs Base64 encoding is a trade-off between readability and space efficiency + // opting for Base64 here to reduce the size of the cluster state + return Base64.getEncoder().encodeToString(hash.digest()); + } + + /** + * Determines the roles to delete by comparing the indexed roles with the roles in the built-in roles. + * @return the set of roles to delete + */ + public static Set determineRolesToDelete(final QueryableBuiltInRoles roles, final Map indexedRolesDigests) { + assert roles != null; + if (indexedRolesDigests == null) { + // nothing indexed, nothing to delete + return Set.of(); + } + final Set rolesToDelete = Sets.difference(indexedRolesDigests.keySet(), roles.rolesDigest().keySet()); + return Collections.unmodifiableSet(rolesToDelete); + } + + /** + * Determines the roles to upsert by comparing the indexed roles and their digests with the current built-in roles. + * @return the set of roles to upsert (create or update) + */ + public static Set determineRolesToUpsert( + final QueryableBuiltInRoles roles, + final Map indexedRolesDigests + ) { + assert roles != null; + final Set rolesToUpsert = new HashSet<>(); + for (RoleDescriptor role : roles.roleDescriptors()) { + final String roleDigest = roles.rolesDigest().get(role.getName()); + if (indexedRolesDigests == null || indexedRolesDigests.containsKey(role.getName()) == false) { + rolesToUpsert.add(role); // a new role to create + } else if (indexedRolesDigests.get(role.getName()).equals(roleDigest) == false) { + rolesToUpsert.add(role); // an existing role that needs to be updated + } + } + return Collections.unmodifiableSet(rolesToUpsert); + } + + private QueryableBuiltInRolesUtils() { + throw new IllegalAccessError("not allowed"); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProvider.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProvider.java new file mode 100644 index 0000000000000..710e94b7ac879 --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProvider.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.common.util.CachedSupplier; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; + +import java.util.Collection; +import java.util.Collections; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * A provider of the built-in reserved roles. + *

+ * This provider fetches all reserved roles from the {@link ReservedRolesStore} and calculates their hashes lazily. + * The reserved roles are static and do not change during runtime, hence this provider will never notify any listeners. + *

+ */ +public final class QueryableReservedRolesProvider implements QueryableBuiltInRoles.Provider { + + private final Supplier reservedRolesSupplier; + + /** + * Constructs a new reserved roles provider. + * + * @param reservedRolesStore the store to fetch the reserved roles from. + * Having a store reference here is necessary to ensure that static fields are initialized. + */ + public QueryableReservedRolesProvider(ReservedRolesStore reservedRolesStore) { + this.reservedRolesSupplier = CachedSupplier.wrap(() -> { + final Collection roleDescriptors = Collections.unmodifiableCollection(ReservedRolesStore.roleDescriptors()); + return new QueryableBuiltInRoles( + roleDescriptors.stream() + .collect(Collectors.toUnmodifiableMap(RoleDescriptor::getName, QueryableBuiltInRolesUtils::calculateHash)), + roleDescriptors + ); + }); + } + + @Override + public QueryableBuiltInRoles getRoles() { + return reservedRolesSupplier.get(); + } + + @Override + public void addListener(QueryableBuiltInRoles.Listener listener) { + // no-op: reserved roles are static and do not change + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index f3222a74b530c..78f7209c06e3a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -586,7 +586,7 @@ private static int readMappingVersion(String indexName, MappingMetadata mappingM * Resolves a concrete index name or alias to a {@link IndexMetadata} instance. Requires * that if supplied with an alias, the alias resolves to at most one concrete index. */ - private static IndexMetadata resolveConcreteIndex(final String indexOrAliasName, final Metadata metadata) { + public static IndexMetadata resolveConcreteIndex(final String indexOrAliasName, final Metadata metadata) { final IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(indexOrAliasName); if (indexAbstraction != null) { final List indices = indexAbstraction.getIndices(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtilsTests.java new file mode 100644 index 0000000000000..5b4787f25ae7f --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesUtilsTests.java @@ -0,0 +1,296 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper; +import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissionGroup; +import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.junit.BeforeClass; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.core.security.support.MetadataUtils.RESERVED_METADATA_KEY; +import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToDelete; +import static org.elasticsearch.xpack.security.support.QueryableBuiltInRolesUtils.determineRolesToUpsert; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class QueryableBuiltInRolesUtilsTests extends ESTestCase { + + @BeforeClass + public static void setupReservedRolesStore() { + new ReservedRolesStore(); // initialize the store + } + + public void testCalculateHash() { + assertThat( + QueryableBuiltInRolesUtils.calculateHash(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR), + equalTo("bWEFdFo4WX229wdhdecfiz5QHMYEssh3ex8hizRgg+Q=") + ); + } + + public void testEmptyOrNullRolesToUpsertOrDelete() { + // test empty roles and index digests + final QueryableBuiltInRoles emptyRoles = new QueryableBuiltInRoles(Map.of(), Set.of()); + assertThat(determineRolesToDelete(emptyRoles, Map.of()), is(empty())); + assertThat(determineRolesToUpsert(emptyRoles, Map.of()), is(empty())); + + // test empty roles and null indexed digests + assertThat(determineRolesToDelete(emptyRoles, null), is(empty())); + assertThat(determineRolesToUpsert(emptyRoles, null), is(empty())); + } + + public void testNoRolesToUpsertOrDelete() { + { + QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor") + ) + ); + + // no roles to delete or upsert since the built-in roles are the same as the indexed roles + assertThat(determineRolesToDelete(currentBuiltInRoles, currentBuiltInRoles.rolesDigest()), is(empty())); + assertThat(determineRolesToUpsert(currentBuiltInRoles, currentBuiltInRoles.rolesDigest()), is(empty())); + } + { + QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor"), + supermanRole("monitor", "read") + ) + ); + + Map digests = buildDigests( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor"), + supermanRole("monitor", "read") + ) + ); + + // no roles to delete or upsert since the built-in roles are the same as the indexed roles + assertThat(determineRolesToDelete(currentBuiltInRoles, digests), is(empty())); + assertThat(determineRolesToUpsert(currentBuiltInRoles, digests), is(empty())); + } + { + final RoleDescriptor randomRole = RoleDescriptorTestHelper.randomRoleDescriptor(); + final QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles(Set.of(randomRole)); + final Map digests = buildDigests( + Set.of( + new RoleDescriptor( + randomRole.getName(), + randomRole.getClusterPrivileges(), + randomRole.getIndicesPrivileges(), + randomRole.getApplicationPrivileges(), + randomRole.getConditionalClusterPrivileges(), + randomRole.getRunAs(), + randomRole.getMetadata(), + randomRole.getTransientMetadata(), + randomRole.getRemoteIndicesPrivileges(), + randomRole.getRemoteClusterPermissions(), + randomRole.getRestriction(), + randomRole.getDescription() + ) + ) + ); + + assertThat(determineRolesToDelete(currentBuiltInRoles, digests), is(empty())); + assertThat(determineRolesToUpsert(currentBuiltInRoles, digests), is(empty())); + } + } + + public void testRolesToDeleteOnly() { + Map indexedDigests = buildDigests( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor"), + supermanRole("monitor", "read", "view_index_metadata", "read_cross_cluster") + ) + ); + + QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor") + ) + ); + + // superman is the only role that needs to be deleted since it is not in a current built-in role + assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), containsInAnyOrder("superman")); + assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), is(empty())); + + // passing empty built-in roles should result in all indexed roles needing to be deleted + QueryableBuiltInRoles emptyBuiltInRoles = new QueryableBuiltInRoles(Map.of(), Set.of()); + assertThat( + determineRolesToDelete(emptyBuiltInRoles, indexedDigests), + containsInAnyOrder("superman", "viewer", "editor", "superuser") + ); + assertThat(determineRolesToUpsert(emptyBuiltInRoles, indexedDigests), is(empty())); + } + + public void testRolesToUpdateOnly() { + Map indexedDigests = buildDigests( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor"), + supermanRole("monitor", "read", "write") + ) + ); + + RoleDescriptor updatedSupermanRole = supermanRole("monitor", "read", "view_index_metadata", "read_cross_cluster"); + QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor"), + updatedSupermanRole + ) + ); + + // superman is the only role that needs to be updated since its definition has changed + assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), is(empty())); + assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), containsInAnyOrder(updatedSupermanRole)); + assertThat(currentBuiltInRoles.rolesDigest().get("superman"), is(not(equalTo(indexedDigests.get("superman"))))); + } + + public void testRolesToCreateOnly() { + Map indexedDigests = buildDigests( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor") + ) + ); + + RoleDescriptor newSupermanRole = supermanRole("monitor", "read", "view_index_metadata", "read_cross_cluster"); + QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor"), + newSupermanRole + ) + ); + + // superman is the only role that needs to be created since it is not in the indexed roles + assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), is(empty())); + assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), containsInAnyOrder(newSupermanRole)); + + // passing empty indexed roles should result in all roles needing to be created + assertThat(determineRolesToDelete(currentBuiltInRoles, Map.of()), is(empty())); + assertThat( + determineRolesToUpsert(currentBuiltInRoles, Map.of()), + containsInAnyOrder(currentBuiltInRoles.roleDescriptors().toArray(new RoleDescriptor[0])) + ); + } + + public void testRolesToUpsertAndDelete() { + Map indexedDigests = buildDigests( + Set.of( + ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, + ReservedRolesStore.roleDescriptor("viewer"), + ReservedRolesStore.roleDescriptor("editor") + ) + ); + + RoleDescriptor newSupermanRole = supermanRole("monitor"); + QueryableBuiltInRoles currentBuiltInRoles = buildQueryableBuiltInRoles( + Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, newSupermanRole) + ); + + // superman is the only role that needs to be updated since its definition has changed + assertThat(determineRolesToDelete(currentBuiltInRoles, indexedDigests), containsInAnyOrder("viewer", "editor")); + assertThat(determineRolesToUpsert(currentBuiltInRoles, indexedDigests), containsInAnyOrder(newSupermanRole)); + } + + private static RoleDescriptor supermanRole(String... indicesPrivileges) { + return new RoleDescriptor( + "superman", + new String[] { "all" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(false).build(), + RoleDescriptor.IndicesPrivileges.builder() + .indices("*") + .privileges(indicesPrivileges) + .allowRestrictedIndices(true) + .build() }, + new RoleDescriptor.ApplicationResourcePrivileges[] { + RoleDescriptor.ApplicationResourcePrivileges.builder().application("*").privileges("*").resources("*").build() }, + null, + new String[] { "*" }, + randomlyOrderedSupermanMetadata(), + Collections.emptyMap(), + new RoleDescriptor.RemoteIndicesPrivileges[] { + new RoleDescriptor.RemoteIndicesPrivileges( + RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(false).build(), + "*" + ), + new RoleDescriptor.RemoteIndicesPrivileges( + RoleDescriptor.IndicesPrivileges.builder() + .indices("*") + .privileges(indicesPrivileges) + .allowRestrictedIndices(true) + .build(), + "*" + ) }, + new RemoteClusterPermissions().addGroup( + new RemoteClusterPermissionGroup( + RemoteClusterPermissions.getSupportedRemoteClusterPermissions().toArray(new String[0]), + new String[] { "*" } + ) + ), + null, + "Grants full access to cluster management and data indices." + ); + } + + private static Map randomlyOrderedSupermanMetadata() { + final LinkedHashMap metadata = new LinkedHashMap<>(); + if (randomBoolean()) { + metadata.put("foo", "bar"); + metadata.put("baz", "qux"); + metadata.put(RESERVED_METADATA_KEY, true); + } else { + metadata.put(RESERVED_METADATA_KEY, true); + metadata.put("foo", "bar"); + metadata.put("baz", "qux"); + } + return metadata; + } + + private static QueryableBuiltInRoles buildQueryableBuiltInRoles(Set roles) { + final Map digests = buildDigests(roles); + return new QueryableBuiltInRoles(digests, roles); + } + + private static Map buildDigests(Set roles) { + final Map digests = new HashMap<>(); + for (RoleDescriptor role : roles) { + digests.put(role.getName(), QueryableBuiltInRolesUtils.calculateHash(role)); + } + return digests; + } +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProviderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProviderTests.java new file mode 100644 index 0000000000000..7beb078795b29 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/QueryableReservedRolesProviderTests.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; + +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; + +public class QueryableReservedRolesProviderTests extends ESTestCase { + + public void testReservedRoleProvider() { + QueryableReservedRolesProvider provider = new QueryableReservedRolesProvider(new ReservedRolesStore()); + assertNotNull(provider.getRoles()); + assertThat(provider.getRoles(), equalTo(provider.getRoles())); + assertThat(provider.getRoles().rolesDigest().size(), equalTo(ReservedRolesStore.roleDescriptors().size())); + assertThat( + provider.getRoles().rolesDigest().keySet(), + equalTo(ReservedRolesStore.roleDescriptors().stream().map(RoleDescriptor::getName).collect(Collectors.toSet())) + ); + } + +} From cf1f2cbc34e97b8d280ee74e824ac1106c24cef0 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Mon, 16 Dec 2024 14:08:11 -0500 Subject: [PATCH 3/4] Unmute ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config (#118788) * Unmute ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config * Revert "Unmute ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config" This reverts commit 48e76936e622072da7c214842613d22b9830dc45. * Unmute test, this time without formatting the entire file --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 2689f02cc92cd..be3805d887bbe 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -40,9 +40,6 @@ tests: - class: org.elasticsearch.packaging.test.WindowsServiceTests method: test33JavaChanged issue: https://github.com/elastic/elasticsearch/issues/113177 -- class: org.elasticsearch.smoketest.MlWithSecurityIT - method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config} - issue: https://github.com/elastic/elasticsearch/issues/108997 - class: org.elasticsearch.packaging.test.WindowsServiceTests method: test80JavaOptsInEnvVar issue: https://github.com/elastic/elasticsearch/issues/113219 From 8e988689438f9579dc23427efc85d320e8289ce3 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 16 Dec 2024 21:26:45 +0100 Subject: [PATCH 4/4] Support multi-index LOOKUP JOIN and various bug fixes (#118429) While working on supporting multi-index LOOKUP JOIN; various bugs were fixed: * Previously we just used '*' for lookup-join indices, because the fieldnames were sometimes not being correctly determined. The problem was with KEEP referencing fields from the right that had previously been defined on the left as aliases, including using the ROW command. We normally don't want to ask for aliases, but if they could be shadowed by a lookup join, we need to keep them. * With both single and multi-index LOOKUP JOIN we need to mark each index as potentially wildcard fields, if the KEEP commands occur before the LOOKUP JOIN. --- .../esql/qa/mixed/MixedClusterEsqlSpecIT.java | 4 +- .../xpack/esql/ccq/MultiClusterSpecIT.java | 8 +- .../rest/RequestIndexFilteringTestCase.java | 11 + .../src/main/resources/lookup-join.csv-spec | 566 +++++++++++++++--- .../src/main/resources/message_types.csv | 2 + .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../xpack/esql/analysis/Analyzer.java | 12 +- .../xpack/esql/analysis/AnalyzerContext.java | 12 +- .../xpack/esql/session/EsqlSession.java | 269 ++++----- .../elasticsearch/xpack/esql/CsvTests.java | 2 +- .../esql/analysis/AnalyzerTestUtils.java | 4 +- .../xpack/esql/analysis/AnalyzerTests.java | 6 +- .../xpack/esql/analysis/VerifierTests.java | 2 +- .../optimizer/LogicalPlanOptimizerTests.java | 37 +- .../optimizer/PhysicalPlanOptimizerTests.java | 4 +- .../session/IndexResolverFieldNamesTests.java | 226 ++++++- 16 files changed, 902 insertions(+), 265 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java index 1120a69cc5166..5efe7ffc800a2 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java @@ -21,7 +21,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V5; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V6; import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.ASYNC; public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase { @@ -96,7 +96,7 @@ protected boolean supportsInferenceTestService() { @Override protected boolean supportsIndexModeLookup() throws IOException { - return hasCapabilities(List.of(JOIN_LOOKUP_V5.capabilityName())); + return hasCapabilities(List.of(JOIN_LOOKUP_V6.capabilityName())); } @Override diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 5c7f981c93a97..dd75776973c3d 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -48,7 +48,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V5; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V6; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC; @@ -124,7 +124,7 @@ protected void shouldSkipTest(String testName) throws IOException { assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V5.capabilityName())); + assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V6.capabilityName())); } private TestFeatureService remoteFeaturesService() throws IOException { @@ -283,8 +283,8 @@ protected boolean supportsInferenceTestService() { @Override protected boolean supportsIndexModeLookup() throws IOException { - // CCS does not yet support JOIN_LOOKUP_V5 and clusters falsely report they have this capability - // return hasCapabilities(List.of(JOIN_LOOKUP_V5.capabilityName())); + // CCS does not yet support JOIN_LOOKUP_V6 and clusters falsely report they have this capability + // return hasCapabilities(List.of(JOIN_LOOKUP_V6.capabilityName())); return false; } } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index 406997b66dbf0..2aae4c94c33fe 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -14,6 +14,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.AssertWarnings; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.junit.After; import org.junit.Assert; @@ -219,6 +220,16 @@ public void testIndicesDontExist() throws IOException { assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("index_not_found_exception")); assertThat(e.getMessage(), containsString("no such index [foo]")); + + if (EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()) { + e = expectThrows( + ResponseException.class, + () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test1 | LOOKUP JOIN foo ON id1")) + ); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + assertThat(e.getMessage(), containsString("verification_exception")); + assertThat(e.getMessage(), containsString("Unknown index [foo]")); + } } private static RestEsqlTestCase.RequestObjectBuilder timestampFilter(String op, String date) throws IOException { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 7fed4f377096f..8b8d24b1bb156 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -3,8 +3,12 @@ // Reuses the sample dataset and commands from enrich.csv-spec // +############################################### +# Tests with languages_lookup index +############################################### + basicOnTheDataNode -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | EVAL language_code = languages @@ -21,7 +25,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; basicRow -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW language_code = 1 | LOOKUP JOIN languages_lookup ON language_code @@ -32,7 +36,7 @@ language_code:integer | language_name:keyword ; basicOnTheCoordinator -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | SORT emp_no @@ -49,7 +53,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; subsequentEvalOnTheDataNode -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | EVAL language_code = languages @@ -67,7 +71,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x ; subsequentEvalOnTheCoordinator -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | SORT emp_no @@ -85,7 +89,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x ; sortEvalBeforeLookup -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | SORT emp_no @@ -102,7 +106,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; nonUniqueLeftKeyOnTheDataNode -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | WHERE emp_no <= 10030 @@ -126,7 +130,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; nonUniqueRightKeyOnTheDataNode -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | EVAL language_code = emp_no % 10 @@ -146,7 +150,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:k ; nonUniqueRightKeyOnTheCoordinator -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | SORT emp_no @@ -166,7 +170,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:k ; nonUniqueRightKeyFromRow -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW language_code = 2 | LOOKUP JOIN languages_lookup_non_unique_key ON language_code @@ -178,8 +182,73 @@ language_code:integer | language_name:keyword | country:keyword 2 | [German, German, German] | [Austria, Germany, Switzerland] ; +############################################### +# Filtering tests with languages_lookup index +############################################### + +lookupWithFilterOnLeftSideField +required_capability: join_lookup_v6 + +FROM employees +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| SORT emp_no +| KEEP emp_no, language_code, language_name +| WHERE emp_no >= 10091 AND emp_no < 10094 +; + +emp_no:integer | language_code:integer | language_name:keyword +10091 | 3 | Spanish +10092 | 1 | English +10093 | 3 | Spanish +; + +lookupMessageWithFilterOnRightSideField-Ignore +required_capability: join_lookup_v6 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| WHERE type == "Error" +| KEEP @timestamp, client_ip, event_duration, message, type +| SORT @timestamp DESC +; + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error +; + +lookupWithFieldAndRightSideAfterStats +required_capability: join_lookup_v6 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| STATS count = count(message) BY type +| WHERE type == "Error" +; + +count:long | type:keyword +3 | Error +; + +lookupWithFieldOnJoinKey-Ignore +required_capability: join_lookup_v6 + +FROM employees +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| WHERE language_code > 1 AND language_name IS NOT NULL +| KEEP emp_no, language_code, language_name +; + +emp_no:integer | language_code:integer | language_name:keyword +10001 | 2 | French +10003 | 4 | German +; + nullJoinKeyOnTheDataNode -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | WHERE emp_no < 10004 @@ -197,7 +266,7 @@ emp_no:integer | language_code:integer | language_name:keyword mvJoinKeyOnTheDataNode -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM employees | WHERE 10003 < emp_no AND emp_no < 10008 @@ -215,7 +284,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; mvJoinKeyFromRow -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW language_code = [4, 5, 6, 7] | LOOKUP JOIN languages_lookup_non_unique_key ON language_code @@ -228,7 +297,7 @@ language_code:integer | language_name:keyword | country:keyword ; mvJoinKeyFromRowExpanded -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW language_code = [4, 5, 6, 7, 8] | MV_EXPAND language_code @@ -245,10 +314,26 @@ language_code:integer | language_name:keyword | country:keyword 8 | Mv-Lang2 | Mv-Land2 ; +############################################### +# Tests with clientips_lookup index +############################################### + lookupIPFromRow -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", right = "right" +| LOOKUP JOIN clientips_lookup ON client_ip +; + +left:keyword | client_ip:keyword | right:keyword | env:keyword +left | 172.21.0.5 | right | Development +; + +lookupIPFromKeepRow +required_capability: join_lookup_v6 ROW left = "left", client_ip = "172.21.0.5", right = "right" +| KEEP left, client_ip, right | LOOKUP JOIN clientips_lookup ON client_ip ; @@ -257,7 +342,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowing -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -268,7 +353,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowingKeep -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -281,7 +366,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowingKeepReordered -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -294,7 +379,7 @@ right | Development | 172.21.0.5 ; lookupIPFromIndex -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -313,7 +398,7 @@ ignoreOrder:true ; lookupIPFromIndexKeep -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -332,8 +417,30 @@ ignoreOrder:true 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | QA ; +lookupIPFromIndexKeepKeep +required_capability: join_lookup_v6 + +FROM sample_data +| KEEP client_ip, event_duration, @timestamp, message +| RENAME @timestamp AS timestamp, message AS msg +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| KEEP timestamp, client_ip, event_duration, msg, env +; +ignoreOrder:true + +timestamp:date | client_ip:keyword | event_duration:long | msg:keyword | env:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Production +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Production +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Production +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Production +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | Development +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | QA +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | QA +; + lookupIPFromIndexStats -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -349,7 +456,7 @@ count:long | env:keyword ; lookupIPFromIndexStatsKeep -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -365,10 +472,43 @@ count:long | env:keyword 1 | Development ; +statsAndLookupIPFromIndex +required_capability: join_lookup_v6 + +FROM sample_data +| EVAL client_ip = client_ip::keyword +| STATS count = count(client_ip) BY client_ip +| LOOKUP JOIN clientips_lookup ON client_ip +| SORT count DESC, client_ip ASC, env ASC +; + +count:long | client_ip:keyword | env:keyword +4 | 172.21.3.15 | Production +1 | 172.21.0.5 | Development +1 | 172.21.2.113 | QA +1 | 172.21.2.162 | QA +; + +############################################### +# Tests with message_types_lookup index +############################################### + lookupMessageFromRow -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 + +ROW left = "left", message = "Connected to 10.1.0.1", right = "right" +| LOOKUP JOIN message_types_lookup ON message +; + +left:keyword | message:keyword | right:keyword | type:keyword +left | Connected to 10.1.0.1 | right | Success +; + +lookupMessageFromKeepRow +required_capability: join_lookup_v6 ROW left = "left", message = "Connected to 10.1.0.1", right = "right" +| KEEP left, message, right | LOOKUP JOIN message_types_lookup ON message ; @@ -377,7 +517,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromRowWithShadowing -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -388,7 +528,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromRowWithShadowingKeep -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -400,7 +540,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromIndex -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -418,7 +558,7 @@ ignoreOrder:true ; lookupMessageFromIndexKeep -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -436,8 +576,28 @@ ignoreOrder:true 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | Success ; +lookupMessageFromIndexKeepKeep +required_capability: join_lookup_v6 + +FROM sample_data +| KEEP client_ip, event_duration, @timestamp, message +| LOOKUP JOIN message_types_lookup ON message +| KEEP @timestamp, client_ip, event_duration, message, type +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | Disconnected +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | Success +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | Success +; + lookupMessageFromIndexKeepReordered -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -456,7 +616,7 @@ Success | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 ; lookupMessageFromIndexStats -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -471,7 +631,7 @@ count:long | type:keyword ; lookupMessageFromIndexStatsKeep -required_capability: join_lookup_v5 +required_capability: join_lookup_v6 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -486,67 +646,333 @@ count:long | type:keyword 1 | Disconnected ; -// -// Filtering tests -// +statsAndLookupMessageFromIndex +required_capability: join_lookup_v6 -lookupWithFilterOnLeftSideField -required_capability: join_lookup_v5 +FROM sample_data +| STATS count = count(message) BY message +| LOOKUP JOIN message_types_lookup ON message +| KEEP count, type, message +| SORT count DESC, message ASC +; -FROM employees -| EVAL language_code = languages -| LOOKUP JOIN languages_lookup ON language_code -| SORT emp_no -| KEEP emp_no, language_code, language_name -| WHERE emp_no >= 10091 AND emp_no < 10094 +count:long | type:keyword | message:keyword +3 | Error | Connection error +1 | Success | Connected to 10.1.0.1 +1 | Success | Connected to 10.1.0.2 +1 | Success | Connected to 10.1.0.3 +1 | Disconnected | Disconnected ; -emp_no:integer | language_code:integer | language_name:keyword -10091 | 3 | Spanish -10092 | 1 | English -10093 | 3 | Spanish +lookupMessageFromIndexTwice +required_capability: join_lookup_v6 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| RENAME message AS message1, type AS type1 +| EVAL message = client_ip::keyword +| LOOKUP JOIN message_types_lookup ON message +| RENAME message AS message2, type AS type2 ; +ignoreOrder:true -lookupMessageWithFilterOnRightSideField-Ignore -required_capability: join_lookup_v5 +@timestamp:date | client_ip:ip | event_duration:long | message1:keyword | type1:keyword | message2:keyword | type2:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success | 172.21.3.15 | null +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error | 172.21.3.15 | null +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error | 172.21.3.15 | null +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error | 172.21.3.15 | null +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | Disconnected | 172.21.0.5 | null +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | Success | 172.21.2.113 | null +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | Success | 172.21.2.162 | null +; + +lookupMessageFromIndexTwiceKeep +required_capability: join_lookup_v6 FROM sample_data | LOOKUP JOIN message_types_lookup ON message -| WHERE type == "Error" -| KEEP @timestamp, client_ip, event_duration, message, type -| SORT @timestamp DESC +| RENAME message AS message1, type AS type1 +| EVAL message = client_ip::keyword +| LOOKUP JOIN message_types_lookup ON message +| RENAME message AS message2, type AS type2 +| KEEP @timestamp, client_ip, event_duration, message1, type1, message2, type2 ; +ignoreOrder:true -@timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword -2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error -2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error -2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error +@timestamp:date | client_ip:ip | event_duration:long | message1:keyword | type1:keyword | message2:keyword | type2:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success | 172.21.3.15 | null +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error | 172.21.3.15 | null +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error | 172.21.3.15 | null +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error | 172.21.3.15 | null +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | Disconnected | 172.21.0.5 | null +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | Success | 172.21.2.113 | null +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | Success | 172.21.2.162 | null ; -lookupWithFieldAndRightSideAfterStats -required_capability: join_lookup_v5 +############################################### +# Tests with clientips_lookup and message_types_lookup indexes +############################################### + +lookupIPAndMessageFromRow +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowKeepBefore +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" +| KEEP left, client_ip, message, right +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowKeepBetween +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" +| LOOKUP JOIN clientips_lookup ON client_ip +| KEEP left, client_ip, message, right, env +| LOOKUP JOIN message_types_lookup ON message +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowKeepAfter +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +| KEEP left, client_ip, message, right, env, type +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowWithShadowing +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", type = "type", right = "right" +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowWithShadowingKeep +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +| KEEP left, client_ip, message, right, env, type +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowWithShadowingKeepKeep +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| KEEP left, client_ip, message, right, env +| LOOKUP JOIN message_types_lookup ON message +| KEEP left, client_ip, message, right, env, type +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowWithShadowingKeepKeepKeep +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" +| EVAL client_ip = client_ip::keyword +| KEEP left, client_ip, message, right, env +| LOOKUP JOIN clientips_lookup ON client_ip +| KEEP left, client_ip, message, right, env +| LOOKUP JOIN message_types_lookup ON message +| KEEP left, client_ip, message, right, env, type +; + +left:keyword | client_ip:keyword | message:keyword | right:keyword | env:keyword | type:keyword +left | 172.21.0.5 | Connected to 10.1.0.1 | right | Development | Success +; + +lookupIPAndMessageFromRowWithShadowingKeepReordered +required_capability: join_lookup_v6 + +ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +| KEEP right, env, type, client_ip +; + +right:keyword | env:keyword | type:keyword | client_ip:keyword +right | Development | Success | 172.21.0.5 +; + +lookupIPAndMessageFromIndex +required_capability: join_lookup_v6 FROM sample_data +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip | LOOKUP JOIN message_types_lookup ON message -| STATS count = count(message) BY type -| WHERE type == "Error" ; +ignoreOrder:true -count:long | type:keyword -3 | Error +@timestamp:date | event_duration:long | message:keyword | client_ip:keyword | env:keyword | type:keyword +2023-10-23T13:55:01.543Z | 1756467 | Connected to 10.1.0.1 | 172.21.3.15 | Production | Success +2023-10-23T13:53:55.832Z | 5033755 | Connection error | 172.21.3.15 | Production | Error +2023-10-23T13:52:55.015Z | 8268153 | Connection error | 172.21.3.15 | Production | Error +2023-10-23T13:51:54.732Z | 725448 | Connection error | 172.21.3.15 | Production | Error +2023-10-23T13:33:34.937Z | 1232382 | Disconnected | 172.21.0.5 | Development | Disconnected +2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 | 172.21.2.113 | QA | Success +2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 | 172.21.2.162 | QA | Success ; -lookupWithFieldOnJoinKey-Ignore -required_capability: join_lookup_v5 +lookupIPAndMessageFromIndexKeep +required_capability: join_lookup_v6 -FROM employees -| EVAL language_code = languages -| LOOKUP JOIN languages_lookup ON language_code -| WHERE language_code > 1 AND language_name IS NOT NULL -| KEEP emp_no, language_code, language_name +FROM sample_data +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +| KEEP @timestamp, client_ip, event_duration, message, env, type ; +ignoreOrder:true -emp_no:integer | language_code:integer | language_name:keyword -10001 | 2 | French -10003 | 4 | German +@timestamp:date | client_ip:keyword | event_duration:long | message:keyword | env:keyword | type:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Production | Success +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Production | Error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Production | Error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Production | Error +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | Development | Disconnected +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | QA | Success +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | QA | Success +; + +lookupIPAndMessageFromIndexStats +required_capability: join_lookup_v6 + +FROM sample_data +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +| STATS count = count(*) BY env, type +| SORT count DESC, env ASC, type ASC +; + +count:long | env:keyword | type:keyword +3 | Production | Error +2 | QA | Success +1 | Development | Disconnected +1 | Production | Success +; + +lookupIPAndMessageFromIndexStatsKeep +required_capability: join_lookup_v6 + +FROM sample_data +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +| KEEP client_ip, env, type +| STATS count = count(*) BY env, type +| SORT count DESC, env ASC, type ASC +; + +count:long | env:keyword | type:keyword +3 | Production | Error +2 | QA | Success +1 | Development | Disconnected +1 | Production | Success +; + +statsAndLookupIPAndMessageFromIndex +required_capability: join_lookup_v6 + +FROM sample_data +| EVAL client_ip = client_ip::keyword +| STATS count = count(*) BY client_ip, message +| LOOKUP JOIN clientips_lookup ON client_ip +| LOOKUP JOIN message_types_lookup ON message +| SORT count DESC, client_ip ASC, message ASC +; + +count:long | client_ip:keyword | message:keyword | env:keyword | type:keyword +3 | 172.21.3.15 | Connection error | Production | Error +1 | 172.21.0.5 | Disconnected | Development | Disconnected +1 | 172.21.2.113 | Connected to 10.1.0.2 | QA | Success +1 | 172.21.2.162 | Connected to 10.1.0.3 | QA | Success +1 | 172.21.3.15 | Connected to 10.1.0.1 | Production | Success +; + +lookupIPAndMessageFromIndexChainedEvalKeep +required_capability: join_lookup_v6 + +FROM sample_data +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| EVAL message = CONCAT(env, " environment") +| LOOKUP JOIN message_types_lookup ON message +| KEEP @timestamp, client_ip, event_duration, message, type +; +ignoreOrder:true + +@timestamp:date | client_ip:keyword | event_duration:long | message:keyword | type:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Production environment | Production +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Production environment | Production +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Production environment | Production +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Production environment | Production +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Development environment | Development +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | QA environment | null +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | QA environment | null +; + +lookupIPAndMessageFromIndexChainedRenameKeep +required_capability: join_lookup_v6 + +FROM sample_data +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| RENAME env AS message +| LOOKUP JOIN message_types_lookup ON message +| KEEP @timestamp, client_ip, event_duration, message, type ; +ignoreOrder:true + +@timestamp:date | client_ip:keyword | event_duration:long | message:keyword | type:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Production | null +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Production | null +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Production | null +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Production | null +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Development | null +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | QA | null +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | QA | null +; + diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv index 8e00485771445..bb4b58046b843 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv @@ -4,3 +4,5 @@ Disconnected,Disconnected Connected to 10.1.0.1,Success Connected to 10.1.0.2,Success Connected to 10.1.0.3,Success +Production environment,Production +Development environment,Development diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index e9a0f89e4f448..235d0dcbe4164 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -547,7 +547,7 @@ public enum Cap { /** * LOOKUP JOIN */ - JOIN_LOOKUP_V5(Build.current().isSnapshot()), + JOIN_LOOKUP_V6(Build.current().isSnapshot()), /** * Fix for https://github.com/elastic/elasticsearch/issues/117054 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index cf91c7df9a034..d59745f03f608 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -198,12 +198,16 @@ private static class ResolveTable extends ParameterizedAnalyzerRule lookupResolution, EnrichResolution enrichResolution ) { // Currently for tests only, since most do not test lookups @@ -26,12 +28,6 @@ public AnalyzerContext( IndexResolution indexResolution, EnrichResolution enrichResolution ) { - this( - configuration, - functionRegistry, - indexResolution, - IndexResolution.invalid("AnalyzerContext constructed without any lookup join resolution"), - enrichResolution - ); + this(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 83480f6651abf..c0290fa2b1d73 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.compute.data.Block; @@ -77,10 +76,12 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -282,10 +283,10 @@ public void analyzedPlan( return; } - TriFunction analyzeAction = (indices, lookupIndices, policies) -> { + Function analyzeAction = (l) -> { planningMetrics.gatherPreAnalysisMetrics(parsed); Analyzer analyzer = new Analyzer( - new AnalyzerContext(configuration, functionRegistry, indices, lookupIndices, policies), + new AnalyzerContext(configuration, functionRegistry, l.indices, l.lookupIndices, l.enrichResolution), verifier ); LogicalPlan plan = analyzer.analyze(parsed); @@ -301,110 +302,77 @@ public void analyzedPlan( EsqlSessionCCSUtils.checkForCcsLicense(indices, indicesExpressionGrouper, verifier.licenseState()); - // TODO: make a separate call for lookup indices final Set targetClusters = enrichPolicyResolver.groupIndicesPerCluster( indices.stream().flatMap(t -> Arrays.stream(Strings.commaDelimitedListToStringArray(t.id().index()))).toArray(String[]::new) ).keySet(); - SubscribableListener.newForked(l -> enrichPolicyResolver.resolvePolicies(targetClusters, unresolvedPolicies, l)) - .andThen((l, enrichResolution) -> { - // we need the match_fields names from enrich policies and THEN, with an updated list of fields, we call field_caps API - var enrichMatchFields = enrichResolution.resolvedEnrichPolicies() - .stream() - .map(ResolvedEnrichPolicy::matchField) - .collect(Collectors.toSet()); - // get the field names from the parsed plan combined with the ENRICH match fields from the ENRICH policy - var fieldNames = fieldNames(parsed, enrichMatchFields); - ListenerResult listenerResult = new ListenerResult(null, null, enrichResolution, fieldNames); - - // first resolve the lookup indices, then the main indices - preAnalyzeLookupIndices(preAnalysis.lookupIndices, listenerResult, l); - }) - .andThen((l, listenerResult) -> { - // resolve the main indices - preAnalyzeIndices(preAnalysis.indices, executionInfo, listenerResult, requestFilter, l); - }) - .andThen((l, listenerResult) -> { - // TODO in follow-PR (for skip_unavailable handling of missing concrete indexes) add some tests for - // invalid index resolution to updateExecutionInfo - if (listenerResult.indices.isValid()) { - // CCS indices and skip_unavailable cluster values can stop the analysis right here - if (analyzeCCSIndices(executionInfo, targetClusters, unresolvedPolicies, listenerResult, logicalPlanListener, l)) - return; - } - // whatever tuple we have here (from CCS-special handling or from the original pre-analysis), pass it on to the next step - l.onResponse(listenerResult); - }) - .andThen((l, listenerResult) -> { - // first attempt (maybe the only one) at analyzing the plan - analyzeAndMaybeRetry(analyzeAction, requestFilter, listenerResult, logicalPlanListener, l); - }) - .andThen((l, listenerResult) -> { - assert requestFilter != null : "The second pre-analysis shouldn't take place when there is no index filter in the request"; - - // "reset" execution information for all ccs or non-ccs (local) clusters, since we are performing the indices - // resolving one more time (the first attempt failed and the query had a filter) - for (String clusterAlias : executionInfo.clusterAliases()) { - executionInfo.swapCluster(clusterAlias, (k, v) -> null); - } - - // here the requestFilter is set to null, performing the pre-analysis after the first step failed - preAnalyzeIndices(preAnalysis.indices, executionInfo, listenerResult, null, l); - }) - .andThen((l, listenerResult) -> { - assert requestFilter != null : "The second analysis shouldn't take place when there is no index filter in the request"; - LOGGER.debug("Analyzing the plan (second attempt, without filter)"); - LogicalPlan plan; - try { - plan = analyzeAction.apply(listenerResult.indices, listenerResult.lookupIndices, listenerResult.enrichResolution); - } catch (Exception e) { - l.onFailure(e); - return; - } - LOGGER.debug("Analyzed plan (second attempt, without filter):\n{}", plan); - l.onResponse(plan); - }) - .addListener(logicalPlanListener); - } + var listener = SubscribableListener.newForked( + l -> enrichPolicyResolver.resolvePolicies(targetClusters, unresolvedPolicies, l) + ).andThen((l, enrichResolution) -> resolveFieldNames(parsed, enrichResolution, l)); + // first resolve the lookup indices, then the main indices + for (TableInfo lookupIndex : preAnalysis.lookupIndices) { + listener = listener.andThen((l, preAnalysisResult) -> { preAnalyzeLookupIndex(lookupIndex, preAnalysisResult, l); }); + } + listener.andThen((l, result) -> { + // resolve the main indices + preAnalyzeIndices(preAnalysis.indices, executionInfo, result, requestFilter, l); + }).andThen((l, result) -> { + // TODO in follow-PR (for skip_unavailable handling of missing concrete indexes) add some tests for + // invalid index resolution to updateExecutionInfo + if (result.indices.isValid()) { + // CCS indices and skip_unavailable cluster values can stop the analysis right here + if (analyzeCCSIndices(executionInfo, targetClusters, unresolvedPolicies, result, logicalPlanListener, l)) return; + } + // whatever tuple we have here (from CCS-special handling or from the original pre-analysis), pass it on to the next step + l.onResponse(result); + }).andThen((l, result) -> { + // first attempt (maybe the only one) at analyzing the plan + analyzeAndMaybeRetry(analyzeAction, requestFilter, result, logicalPlanListener, l); + }).andThen((l, result) -> { + assert requestFilter != null : "The second pre-analysis shouldn't take place when there is no index filter in the request"; + + // "reset" execution information for all ccs or non-ccs (local) clusters, since we are performing the indices + // resolving one more time (the first attempt failed and the query had a filter) + for (String clusterAlias : executionInfo.clusterAliases()) { + executionInfo.swapCluster(clusterAlias, (k, v) -> null); + } - private void preAnalyzeLookupIndices(List indices, ListenerResult listenerResult, ActionListener listener) { - if (indices.size() > 1) { - // Note: JOINs on more than one index are not yet supported - listener.onFailure(new MappingException("More than one LOOKUP JOIN is not supported")); - } else if (indices.size() == 1) { - TableInfo tableInfo = indices.get(0); - TableIdentifier table = tableInfo.id(); - // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types - indexResolver.resolveAsMergedMapping( - table.index(), - Set.of("*"), // TODO: for LOOKUP JOIN, this currently declares all lookup index fields relevant and might fetch too many. - null, - listener.map(indexResolution -> listenerResult.withLookupIndexResolution(indexResolution)) - ); - // TODO: Verify that the resolved index actually has indexMode: "lookup" - } else { + // here the requestFilter is set to null, performing the pre-analysis after the first step failed + preAnalyzeIndices(preAnalysis.indices, executionInfo, result, null, l); + }).andThen((l, result) -> { + assert requestFilter != null : "The second analysis shouldn't take place when there is no index filter in the request"; + LOGGER.debug("Analyzing the plan (second attempt, without filter)"); + LogicalPlan plan; try { - // No lookup indices specified - listener.onResponse( - new ListenerResult( - listenerResult.indices, - IndexResolution.invalid("[none specified]"), - listenerResult.enrichResolution, - listenerResult.fieldNames - ) - ); - } catch (Exception ex) { - listener.onFailure(ex); + plan = analyzeAction.apply(result); + } catch (Exception e) { + l.onFailure(e); + return; } - } + LOGGER.debug("Analyzed plan (second attempt, without filter):\n{}", plan); + l.onResponse(plan); + }).addListener(logicalPlanListener); + } + + private void preAnalyzeLookupIndex(TableInfo tableInfo, PreAnalysisResult result, ActionListener listener) { + TableIdentifier table = tableInfo.id(); + Set fieldNames = result.wildcardJoinIndices().contains(table.index()) ? IndexResolver.ALL_FIELDS : result.fieldNames; + // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types + indexResolver.resolveAsMergedMapping( + table.index(), + fieldNames, + null, + listener.map(indexResolution -> result.addLookupIndexResolution(table.index(), indexResolution)) + ); + // TODO: Verify that the resolved index actually has indexMode: "lookup" } private void preAnalyzeIndices( List indices, EsqlExecutionInfo executionInfo, - ListenerResult listenerResult, + PreAnalysisResult result, QueryBuilder requestFilter, - ActionListener listener + ActionListener listener ) { // TODO we plan to support joins in the future when possible, but for now we'll just fail early if we see one if (indices.size() > 1) { @@ -412,7 +380,7 @@ private void preAnalyzeIndices( listener.onFailure(new MappingException("Queries with multiple indices are not supported")); } else if (indices.size() == 1) { // known to be unavailable from the enrich policy API call - Map unavailableClusters = listenerResult.enrichResolution.getUnavailableClusters(); + Map unavailableClusters = result.enrichResolution.getUnavailableClusters(); TableInfo tableInfo = indices.get(0); TableIdentifier table = tableInfo.id(); @@ -445,34 +413,20 @@ private void preAnalyzeIndices( String indexExpressionToResolve = EsqlSessionCCSUtils.createIndexExpressionFromAvailableClusters(executionInfo); if (indexExpressionToResolve.isEmpty()) { // if this was a pure remote CCS request (no local indices) and all remotes are offline, return an empty IndexResolution - listener.onResponse( - new ListenerResult( - IndexResolution.valid(new EsIndex(table.index(), Map.of(), Map.of())), - listenerResult.lookupIndices, - listenerResult.enrichResolution, - listenerResult.fieldNames - ) - ); + listener.onResponse(result.withIndexResolution(IndexResolution.valid(new EsIndex(table.index(), Map.of(), Map.of())))); } else { // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types indexResolver.resolveAsMergedMapping( indexExpressionToResolve, - listenerResult.fieldNames, + result.fieldNames, requestFilter, - listener.map(indexResolution -> listenerResult.withIndexResolution(indexResolution)) + listener.map(indexResolution -> result.withIndexResolution(indexResolution)) ); } } else { try { // occurs when dealing with local relations (row a = 1) - listener.onResponse( - new ListenerResult( - IndexResolution.invalid("[none specified]"), - listenerResult.lookupIndices, - listenerResult.enrichResolution, - listenerResult.fieldNames - ) - ); + listener.onResponse(result.withIndexResolution(IndexResolution.invalid("[none specified]"))); } catch (Exception ex) { listener.onFailure(ex); } @@ -483,11 +437,11 @@ private boolean analyzeCCSIndices( EsqlExecutionInfo executionInfo, Set targetClusters, Set unresolvedPolicies, - ListenerResult listenerResult, + PreAnalysisResult result, ActionListener logicalPlanListener, - ActionListener l + ActionListener l ) { - IndexResolution indexResolution = listenerResult.indices; + IndexResolution indexResolution = result.indices; EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); EsqlSessionCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.unavailableClusters()); if (executionInfo.isCrossClusterSearch() && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) == 0) { @@ -509,7 +463,7 @@ private boolean analyzeCCSIndices( enrichPolicyResolver.resolvePolicies( newClusters, unresolvedPolicies, - l.map(enrichResolution -> listenerResult.withEnrichResolution(enrichResolution)) + l.map(enrichResolution -> result.withEnrichResolution(enrichResolution)) ); return true; } @@ -517,11 +471,11 @@ private boolean analyzeCCSIndices( } private static void analyzeAndMaybeRetry( - TriFunction analyzeAction, + Function analyzeAction, QueryBuilder requestFilter, - ListenerResult listenerResult, + PreAnalysisResult result, ActionListener logicalPlanListener, - ActionListener l + ActionListener l ) { LogicalPlan plan = null; var filterPresentMessage = requestFilter == null ? "without" : "with"; @@ -529,7 +483,7 @@ private static void analyzeAndMaybeRetry( LOGGER.debug("Analyzing the plan ({} attempt, {} filter)", attemptMessage, filterPresentMessage); try { - plan = analyzeAction.apply(listenerResult.indices, listenerResult.lookupIndices, listenerResult.enrichResolution); + plan = analyzeAction.apply(result); } catch (Exception e) { if (e instanceof VerificationException ve) { LOGGER.debug( @@ -544,7 +498,7 @@ private static void analyzeAndMaybeRetry( } else { // interested only in a VerificationException, but this time we are taking out the index filter // to try and make the index resolution work without any index filtering. In the next step... to be continued - l.onResponse(listenerResult); + l.onResponse(result); } } else { // if the query failed with any other type of exception, then just pass the exception back to the user @@ -557,10 +511,24 @@ private static void analyzeAndMaybeRetry( logicalPlanListener.onResponse(plan); } - static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchFields) { + private static void resolveFieldNames(LogicalPlan parsed, EnrichResolution enrichResolution, ActionListener l) { + try { + // we need the match_fields names from enrich policies and THEN, with an updated list of fields, we call field_caps API + var enrichMatchFields = enrichResolution.resolvedEnrichPolicies() + .stream() + .map(ResolvedEnrichPolicy::matchField) + .collect(Collectors.toSet()); + // get the field names from the parsed plan combined with the ENRICH match fields from the ENRICH policy + l.onResponse(fieldNames(parsed, enrichMatchFields, new PreAnalysisResult(enrichResolution))); + } catch (Exception ex) { + l.onFailure(ex); + } + } + + static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicyMatchFields, PreAnalysisResult result) { if (false == parsed.anyMatch(plan -> plan instanceof Aggregate || plan instanceof Project)) { // no explicit columns selection, for example "from employees" - return IndexResolver.ALL_FIELDS; + return result.withFieldNames(IndexResolver.ALL_FIELDS); } Holder projectAll = new Holder<>(false); @@ -571,7 +539,7 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF projectAll.set(true); }); if (projectAll.get()) { - return IndexResolver.ALL_FIELDS; + return result.withFieldNames(IndexResolver.ALL_FIELDS); } AttributeSet references = new AttributeSet(); @@ -579,6 +547,7 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF // ie "from test | eval lang = languages + 1 | keep *l" should consider both "languages" and "*l" as valid fields to ask for AttributeSet keepCommandReferences = new AttributeSet(); AttributeSet keepJoinReferences = new AttributeSet(); + Set wildcardJoinIndices = new java.util.HashSet<>(); parsed.forEachDown(p -> {// go over each plan top-down if (p instanceof RegexExtract re) { // for Grok and Dissect @@ -596,10 +565,16 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF enrichRefs.removeIf(attr -> attr instanceof EmptyAttribute); references.addAll(enrichRefs); } else if (p instanceof LookupJoin join) { - keepJoinReferences.addAll(join.config().matchFields()); // TODO: why is this empty if (join.config().type() instanceof JoinTypes.UsingJoinType usingJoinType) { keepJoinReferences.addAll(usingJoinType.columns()); } + if (keepCommandReferences.isEmpty()) { + // No KEEP commands after the JOIN, so we need to mark this index for "*" field resolution + wildcardJoinIndices.add(((UnresolvedRelation) join.right()).table().index()); + } else { + // Keep commands can reference the join columns with names that shadow aliases, so we block their removal + keepJoinReferences.addAll(keepCommandReferences); + } } else { references.addAll(p.references()); if (p instanceof UnresolvedRelation ur && ur.indexMode() == IndexMode.TIME_SERIES) { @@ -634,6 +609,10 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF }); // Add JOIN ON column references afterward to avoid Alias removal references.addAll(keepJoinReferences); + // If any JOIN commands need wildcard field-caps calls, persist the index names + if (wildcardJoinIndices.isEmpty() == false) { + result = result.withWildcardJoinIndices(wildcardJoinIndices); + } // remove valid metadata attributes because they will be filtered out by the IndexResolver anyway // otherwise, in some edge cases, we will fail to ask for "*" (all fields) instead @@ -642,12 +621,12 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) { // there cannot be an empty list of fields, we'll ask the simplest and lightest one instead: _index - return IndexResolver.INDEX_METADATA_FIELD; + return result.withFieldNames(IndexResolver.INDEX_METADATA_FIELD); } else { fieldNames.addAll(subfields(fieldNames)); fieldNames.addAll(enrichPolicyMatchFields); fieldNames.addAll(subfields(enrichPolicyMatchFields)); - return fieldNames; + return result.withFieldNames(fieldNames); } } @@ -706,22 +685,36 @@ public PhysicalPlan optimizedPhysicalPlan(LogicalPlan optimizedPlan) { return plan; } - private record ListenerResult( + record PreAnalysisResult( IndexResolution indices, - IndexResolution lookupIndices, + Map lookupIndices, EnrichResolution enrichResolution, - Set fieldNames + Set fieldNames, + Set wildcardJoinIndices ) { - ListenerResult withEnrichResolution(EnrichResolution newEnrichResolution) { - return new ListenerResult(indices(), lookupIndices(), newEnrichResolution, fieldNames()); + PreAnalysisResult(EnrichResolution newEnrichResolution) { + this(null, new HashMap<>(), newEnrichResolution, Set.of(), Set.of()); } - ListenerResult withIndexResolution(IndexResolution newIndexResolution) { - return new ListenerResult(newIndexResolution, lookupIndices(), enrichResolution(), fieldNames()); + PreAnalysisResult withEnrichResolution(EnrichResolution newEnrichResolution) { + return new PreAnalysisResult(indices(), lookupIndices(), newEnrichResolution, fieldNames(), wildcardJoinIndices()); } - ListenerResult withLookupIndexResolution(IndexResolution newIndexResolution) { - return new ListenerResult(indices(), newIndexResolution, enrichResolution(), fieldNames()); + PreAnalysisResult withIndexResolution(IndexResolution newIndexResolution) { + return new PreAnalysisResult(newIndexResolution, lookupIndices(), enrichResolution(), fieldNames(), wildcardJoinIndices()); } - }; + + PreAnalysisResult addLookupIndexResolution(String index, IndexResolution newIndexResolution) { + lookupIndices.put(index, newIndexResolution); + return this; + } + + PreAnalysisResult withFieldNames(Set newFields) { + return new PreAnalysisResult(indices(), lookupIndices(), enrichResolution(), newFields, wildcardJoinIndices()); + } + + public PreAnalysisResult withWildcardJoinIndices(Set wildcardJoinIndices) { + return new PreAnalysisResult(indices(), lookupIndices(), enrichResolution(), fieldNames(), wildcardJoinIndices); + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index c11ef8615eb72..f553c15ef69fa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -263,7 +263,7 @@ public final void test() throws Throwable { ); assumeFalse( "lookup join disabled for csv tests", - testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V5.capabilityName()) + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V6.capabilityName()) ); assumeFalse( "can't use TERM function in csv tests", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index 5e79e40b7e938..85dd36ba0aaa5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -123,8 +123,8 @@ public static IndexResolution expandedDefaultIndexResolution() { return loadMapping("mapping-default.json", "test"); } - public static IndexResolution defaultLookupResolution() { - return loadMapping("mapping-languages.json", "languages_lookup", IndexMode.LOOKUP); + public static Map defaultLookupResolution() { + return Map.of("languages_lookup", loadMapping("mapping-languages.json", "languages_lookup", IndexMode.LOOKUP)); } public static EnrichResolution defaultEnrichResolution() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index cfff245b19244..4e02119b31744 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2139,7 +2139,7 @@ public void testLookupMatchTypeWrong() { } public void testLookupJoinUnknownIndex() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); String errorMessage = "Unknown index [foobar]"; IndexResolution missingLookupIndex = IndexResolution.invalid(errorMessage); @@ -2149,7 +2149,7 @@ public void testLookupJoinUnknownIndex() { EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), analyzerDefaultMapping(), - missingLookupIndex, + Map.of("foobar", missingLookupIndex), defaultEnrichResolution() ), TEST_VERIFIER @@ -2168,7 +2168,7 @@ public void testLookupJoinUnknownIndex() { } public void testLookupJoinUnknownField() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); String query = "FROM test | LOOKUP JOIN languages_lookup ON last_name"; String errorMessage = "1:45: Unknown column [last_name] in right side of join"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 4b916106165fb..58180aafedc0b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1964,7 +1964,7 @@ public void testSortByAggregate() { } public void testLookupJoinDataTypeMismatch() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); query("FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 1d10ebab267ce..c4d7b30115c2d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -219,11 +219,6 @@ public static void init() { enrichResolution = new EnrichResolution(); AnalyzerTestUtils.loadEnrichPolicyResolution(enrichResolution, "languages_idx", "id", "languages_idx", "mapping-languages.json"); - var lookupMapping = loadMapping("mapping-languages.json"); - IndexResolution lookupResolution = IndexResolution.valid( - new EsIndex("language_code", lookupMapping, Map.of("language_code", IndexMode.LOOKUP)) - ); - // Most tests used data from the test index, so we load it here, and use it in the plan() function. mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); @@ -4911,7 +4906,7 @@ public void testPlanSanityCheck() throws Exception { } public void testPlanSanityCheckWithBinaryPlans() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); var plan = optimizedPlan(""" FROM test @@ -5913,15 +5908,15 @@ public void testLookupStats() { * | \_Limit[1000[INTEGER]] * | \_Filter[languages{f}#10 > 1[INTEGER]] * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] - * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); String query = """ FROM test | RENAME languages AS language_code - | LOOKUP JOIN language_code ON language_code + | LOOKUP JOIN languages_lookup ON language_code | WHERE language_code > 1 """; var plan = optimizedPlan(query); @@ -5956,15 +5951,15 @@ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { * | \_Limit[1000[INTEGER]] * | \_Filter[emp_no{f}#7 > 1[INTEGER]] * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] - * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownFilterOnLeftSideField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); String query = """ FROM test | RENAME languages AS language_code - | LOOKUP JOIN language_code ON language_code + | LOOKUP JOIN languages_lookup ON language_code | WHERE emp_no > 1 """; @@ -6000,15 +5995,15 @@ public void testLookupJoinPushDownFilterOnLeftSideField() { * |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]] * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] - * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownDisabledForLookupField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); String query = """ FROM test | RENAME languages AS language_code - | LOOKUP JOIN language_code ON language_code + | LOOKUP JOIN languages_lookup ON language_code | WHERE language_name == "English" """; @@ -6045,15 +6040,15 @@ public void testLookupJoinPushDownDisabledForLookupField() { * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]] * | \_Filter[emp_no{f}#8 > 1[INTEGER]] * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] - * \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20] */ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); String query = """ FROM test | RENAME languages AS language_code - | LOOKUP JOIN language_code ON language_code + | LOOKUP JOIN languages_lookup ON language_code | WHERE language_name == "English" AND emp_no > 1 """; @@ -6098,15 +6093,15 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel * |_EsqlProject[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]] * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] - * \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20] + * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20] */ public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); String query = """ FROM test | RENAME languages AS language_code - | LOOKUP JOIN language_code ON language_code + | LOOKUP JOIN languages_lookup ON language_code | WHERE language_name == "English" OR emp_no > 1 """; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index d0c7a1cd61010..9f6ef89008a24 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -285,7 +285,7 @@ TestDataSource makeTestDataSource( String indexName, String mappingFileName, EsqlFunctionRegistry functionRegistry, - IndexResolution lookupResolution, + Map lookupResolution, EnrichResolution enrichResolution, SearchStats stats ) { @@ -2331,7 +2331,7 @@ public void testVerifierOnMissingReferences() { } public void testVerifierOnMissingReferencesWithBinaryPlans() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V5.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V6.isEnabled()); // Do not assert serialization: // This will have a LookupJoinExec, which is not serializable because it doesn't leave the coordinator. diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 0fe89b24dfc6a..e4271a0a6ddd5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1316,25 +1316,25 @@ public void testCountStar() { } public void testEnrichOnDefaultFieldWithKeep() { - Set fieldNames = EsqlSession.fieldNames(parser.createStatement(""" + Set fieldNames = fieldNames(""" from employees | enrich languages_policy - | keep emp_no"""), Set.of("language_name")); + | keep emp_no""", Set.of("language_name")); assertThat(fieldNames, equalTo(Set.of("emp_no", "emp_no.*", "language_name", "language_name.*"))); } public void testDissectOverwriteName() { - Set fieldNames = EsqlSession.fieldNames(parser.createStatement(""" + Set fieldNames = fieldNames(""" from employees | dissect first_name "%{first_name} %{more}" - | keep emp_no, first_name, more"""), Set.of()); + | keep emp_no, first_name, more""", Set.of()); assertThat(fieldNames, equalTo(Set.of("emp_no", "emp_no.*", "first_name", "first_name.*"))); } public void testEnrichOnDefaultField() { - Set fieldNames = EsqlSession.fieldNames(parser.createStatement(""" + Set fieldNames = fieldNames(""" from employees - | enrich languages_policy"""), Set.of("language_name")); + | enrich languages_policy""", Set.of("language_name")); assertThat(fieldNames, equalTo(ALL_FIELDS)); } @@ -1345,7 +1345,7 @@ public void testMetrics() { assertThat(e.getMessage(), containsString("line 1:1: mismatched input 'METRICS' expecting {")); return; } - Set fieldNames = EsqlSession.fieldNames(parser.createStatement(query), Set.of()); + Set fieldNames = fieldNames(query, Set.of()); assertThat( fieldNames, equalTo( @@ -1363,8 +1363,218 @@ public void testMetrics() { ); } + public void testLookupJoin() { + assertFieldNames( + "FROM employees | KEEP languages | RENAME languages AS language_code | LOOKUP JOIN languages_lookup ON language_code", + Set.of("languages", "languages.*", "language_code", "language_code.*"), + Set.of("languages_lookup") // Since we have KEEP before the LOOKUP JOIN we need to wildcard the lookup index + ); + } + + public void testLookupJoinKeep() { + assertFieldNames( + """ + FROM employees + | KEEP languages + | RENAME languages AS language_code + | LOOKUP JOIN languages_lookup ON language_code + | KEEP languages, language_code, language_name""", + Set.of("languages", "languages.*", "language_code", "language_code.*", "language_name", "language_name.*"), + Set.of() // Since we have KEEP after the LOOKUP, we can use the global field names instead of wildcarding the lookup index + ); + } + + public void testLookupJoinKeepWildcard() { + assertFieldNames( + """ + FROM employees + | KEEP languages + | RENAME languages AS language_code + | LOOKUP JOIN languages_lookup ON language_code + | KEEP language*""", + Set.of("language*", "languages", "languages.*", "language_code", "language_code.*"), + Set.of() // Since we have KEEP after the LOOKUP, we can use the global field names instead of wildcarding the lookup index + ); + } + + public void testMultiLookupJoin() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | LOOKUP JOIN message_types_lookup ON message""", + Set.of("*"), // With no KEEP we should keep all fields + Set.of() // since global field names are wildcarded, we don't need to wildcard any indices + ); + } + + public void testMultiLookupJoinKeepBefore() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | KEEP @timestamp, client_ip, event_duration, message + | LOOKUP JOIN clientips_lookup ON client_ip + | LOOKUP JOIN message_types_lookup ON message""", + Set.of("@timestamp", "@timestamp.*", "client_ip", "client_ip.*", "event_duration", "event_duration.*", "message", "message.*"), + Set.of("clientips_lookup", "message_types_lookup") // Since the KEEP is before both JOINS we need to wildcard both indices + ); + } + + public void testMultiLookupJoinKeepBetween() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | KEEP @timestamp, client_ip, event_duration, message, env + | LOOKUP JOIN message_types_lookup ON message""", + Set.of( + "@timestamp", + "@timestamp.*", + "client_ip", + "client_ip.*", + "event_duration", + "event_duration.*", + "message", + "message.*", + "env", + "env.*" + ), + Set.of("message_types_lookup") // Since the KEEP is before the second JOIN, we need to wildcard the second index + ); + } + + public void testMultiLookupJoinKeepAfter() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | LOOKUP JOIN message_types_lookup ON message + | KEEP @timestamp, client_ip, event_duration, message, env, type""", + Set.of( + "@timestamp", + "@timestamp.*", + "client_ip", + "client_ip.*", + "event_duration", + "event_duration.*", + "message", + "message.*", + "env", + "env.*", + "type", + "type.*" + ), + Set.of() // Since the KEEP is after both JOINs, we can use the global field names + ); + } + + public void testMultiLookupJoinKeepAfterWildcard() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | LOOKUP JOIN message_types_lookup ON message + | KEEP *env*, *type*""", + Set.of("*env*", "*type*", "client_ip", "client_ip.*", "message", "message.*"), + Set.of() // Since the KEEP is after both JOINs, we can use the global field names + ); + } + + public void testMultiLookupJoinSameIndex() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | EVAL client_ip = message + | LOOKUP JOIN clientips_lookup ON client_ip""", + Set.of("*"), // With no KEEP we should keep all fields + Set.of() // since global field names are wildcarded, we don't need to wildcard any indices + ); + } + + public void testMultiLookupJoinSameIndexKeepBefore() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | KEEP @timestamp, client_ip, event_duration, message + | LOOKUP JOIN clientips_lookup ON client_ip + | EVAL client_ip = message + | LOOKUP JOIN clientips_lookup ON client_ip""", + Set.of("@timestamp", "@timestamp.*", "client_ip", "client_ip.*", "event_duration", "event_duration.*", "message", "message.*"), + Set.of("clientips_lookup") // Since there is no KEEP after the last JOIN, we need to wildcard the index + ); + } + + public void testMultiLookupJoinSameIndexKeepBetween() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | KEEP @timestamp, client_ip, event_duration, message, env + | EVAL client_ip = message + | LOOKUP JOIN clientips_lookup ON client_ip""", + Set.of( + "@timestamp", + "@timestamp.*", + "client_ip", + "client_ip.*", + "event_duration", + "event_duration.*", + "message", + "message.*", + "env", + "env.*" + ), + Set.of("clientips_lookup") // Since there is no KEEP after the last JOIN, we need to wildcard the index + ); + } + + public void testMultiLookupJoinSameIndexKeepAfter() { + assertFieldNames( + """ + FROM sample_data + | EVAL client_ip = client_ip::keyword + | LOOKUP JOIN clientips_lookup ON client_ip + | EVAL client_ip = message + | LOOKUP JOIN clientips_lookup ON client_ip + | KEEP @timestamp, client_ip, event_duration, message, env""", + Set.of( + "@timestamp", + "@timestamp.*", + "client_ip", + "client_ip.*", + "event_duration", + "event_duration.*", + "message", + "message.*", + "env", + "env.*" + ), + Set.of() // Since the KEEP is after both JOINs, we can use the global field names + ); + } + + private Set fieldNames(String query, Set enrichPolicyMatchFields) { + var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); + return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); + } + private void assertFieldNames(String query, Set expected) { - Set fieldNames = EsqlSession.fieldNames(parser.createStatement(query), Collections.emptySet()); + Set fieldNames = fieldNames(query, Collections.emptySet()); assertThat(fieldNames, equalTo(expected)); } + + private void assertFieldNames(String query, Set expected, Set wildCardIndices) { + var preAnalysisResult = EsqlSession.fieldNames(parser.createStatement(query), Set.of(), new EsqlSession.PreAnalysisResult(null)); + assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); + assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); + } }