Test DLS and FLS against runtime fields

x-pack/plugin/runtime-fields/qa/with-security/build.gradle

@@ -0,0 +1,26 @@
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
+}
+
+def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
+                          password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
+
+integTest {
+  systemProperty 'tests.rest.cluster.username', clusterCredentials.username
+  systemProperty 'tests.rest.cluster.password', clusterCredentials.password
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user clusterCredentials
+  user username: "test", password: "x-pack-test-password", role: "test"
+}

x-pack/plugin/runtime-fields/qa/with-security/roles.yml

@@ -0,0 +1,13 @@
+test:
+  indices:
+    - names: [ 'dls' ]
+      privileges:
+        - read
+      query: "{\"match\": {\"year\": 2016}}"
+    - names: [ 'fls' ]
+      privileges:
+        - read
+      field_security:
+        grant: [ '*' ]
+        except: [ 'year', 'hidden' ]
+

x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java

@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public class PermissionsIT extends ESRestTestCase {
+
+    private static HighLevelClient highLevelClient;
+    private static HighLevelClient adminHighLevelClient;
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("test", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Before
+    public void initHighLevelClient() {
+        if (highLevelClient == null) {
+            highLevelClient = new HighLevelClient(client());
+            adminHighLevelClient = new HighLevelClient(adminClient());
+        }
+    }
+
+    @AfterClass
+    public static void closeHighLevelClients() throws IOException {
+        highLevelClient.close();
+        adminHighLevelClient.close();
+        highLevelClient = null;
+        adminHighLevelClient = null;
+    }
+
+    public void testDLS() throws IOException  {
+        Request createIndex = new Request("PUT", "/dls");
+        createIndex.setJsonEntity("{\n" +
+            "    \"mappings\" : {\n" +
+            "        \"properties\" : {\n" +
+            "            \"date\" : {\"type\" : \"keyword\"},\n" +
+            "            \"year\" : {\n" +
+            "              \"type\" : \"runtime_script\", \n" +
+            "              \"runtime_type\" : \"keyword\",\n" +
+            "              \"script\" : \"emitValue(doc[\\\"date\\\"].value.substring(0,4))\"\n" +
+            "            }\n" +
+            "        }\n" +
+            "    }\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(createIndex));
+
+        Request indexDoc1 = new Request("PUT", "/dls/_doc/1");
+        indexDoc1.setJsonEntity("{\n" +
+            "    \"date\" : \"2009-11-15T14:12:12\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc1));
+
+        Request indexDoc2 = new Request("PUT", "/dls/_doc/2");
+        indexDoc2.setJsonEntity("{\n" +
+            "    \"date\" : \"2016-11-15T14:12:12\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc2));
+
+        Request indexDoc3 = new Request("PUT", "/dls/_doc/3");
+        indexDoc3.addParameter("refresh", "wait_for");
+        indexDoc3.setJsonEntity("{\n" +
+            "    \"date\" : \"2018-11-15T14:12:12\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc3));
+
+        SearchRequest searchRequest = new SearchRequest("dls");
+        {
+            SearchResponse searchResponse = adminHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
+            assertEquals(3, searchResponse.getHits().getTotalHits().value);
+        }
+        {
+            SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
+            assertEquals(1, searchResponse.getHits().getTotalHits().value);
+        }
+    }
+
+    public void testFLSProtectsData() throws IOException {
+        Request createIndex = new Request("PUT", "/fls");
+        createIndex.setJsonEntity("{\n" +
+            "    \"mappings\" : {\n" +
+            "        \"properties\" : {\n" +
+            "            \"hidden\" : {\"type\" : \"keyword\"},\n" +
+            "            \"hidden_values_count\" : {\n" +
+            "              \"type\" : \"runtime_script\", \n" +
+            "              \"runtime_type\" : \"long\",\n" +
+            "              \"script\" : \"emitValue(doc[\\\"hidden\\\"].size())\"\n" +
+            "            }\n" +
+            "        }\n" +
+            "    }\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(createIndex));
+
+        Request indexDoc1 = new Request("PUT", "/fls/_doc/1");
+        indexDoc1.setJsonEntity("{\n" +
+            "    \"hidden\" : \"should not be read\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc1));
+
+        Request indexDoc2 = new Request("PUT", "/fls/_doc/2");
+        indexDoc2.setJsonEntity("{\n" +
+            "    \"hidden\" : \"should not be read\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc2));
+
+        Request indexDoc3 = new Request("PUT", "/fls/_doc/3");
+        indexDoc3.addParameter("refresh", "wait_for");
+        indexDoc3.setJsonEntity("{\n" +
+            "    \"hidden\" : \"should not be read\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc3));
+
+        SearchRequest searchRequest = new SearchRequest("fls").source(new SearchSourceBuilder().docValueField("hidden_values_count"));
+        {
+            SearchResponse searchResponse = adminHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
+            assertEquals(3, searchResponse.getHits().getTotalHits().value);
+            for (SearchHit hit : searchResponse.getHits().getHits()) {
+                assertEquals(1, hit.getFields().size());
+                assertEquals(1, (int)hit.getFields().get("hidden_values_count").getValue());
+            }
+        }
+        {
+            SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
+            assertEquals(3, searchResponse.getHits().getTotalHits().value);
+            for (SearchHit hit : searchResponse.getHits().getHits()) {
+                assertEquals(0, (int)hit.getFields().get("hidden_values_count").getValue());
+            }
+        }
+    }
+
+    public void testFLSOnRuntimeField() throws IOException {
+        Request createIndex = new Request("PUT", "/fls");
+        createIndex.setJsonEntity("{\n" +
+            "    \"mappings\" : {\n" +
+            "        \"properties\" : {\n" +
+            "            \"date\" : {\"type\" : \"keyword\"},\n" +
+            "            \"year\" : {\n" +
+            "              \"type\" : \"runtime_script\", \n" +
+            "              \"runtime_type\" : \"keyword\",\n" +
+            "              \"script\" : \"emitValue(doc[\\\"date\\\"].value.substring(0,4))\"\n" +
+            "            }\n" +
+            "        }\n" +
+            "    }\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(createIndex));
+
+        Request indexDoc1 = new Request("PUT", "/fls/_doc/1");
+        indexDoc1.setJsonEntity("{\n" +
+            "    \"date\" : \"2009-11-15T14:12:12\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc1));
+
+        Request indexDoc2 = new Request("PUT", "/fls/_doc/2");
+        indexDoc2.setJsonEntity("{\n" +
+            "    \"date\" : \"2016-11-15T14:12:12\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc2));
+
+        Request indexDoc3 = new Request("PUT", "/fls/_doc/3");
+        indexDoc3.addParameter("refresh", "wait_for");
+        indexDoc3.setJsonEntity("{\n" +
+            "    \"date\" : \"2018-11-15T14:12:12\"\n" +
+            "}\n");
+        assertOK(adminClient().performRequest(indexDoc3));
+
+        //There is no FLS directly on runtime fields
+        SearchRequest searchRequest = new SearchRequest("fls").source(new SearchSourceBuilder().docValueField("year"));
+        SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
+        assertEquals(3, searchResponse.getHits().getTotalHits().value);
+        for (SearchHit hit : searchResponse.getHits().getHits()) {
+            Map<String, DocumentField> fields = hit.getFields();
+            assertEquals(1, fields.size());
+            switch(hit.getId()) {
+                case "1":
+                    assertEquals("2009", fields.get("year").getValue().toString());
+                    break;
+                case "2":
+                    assertEquals("2016", fields.get("year").getValue().toString());
+                    break;
+                case "3":
+                    assertEquals("2018", fields.get("year").getValue().toString());
+                    break;
+                default:
+                    throw new UnsupportedOperationException();
+            }
+        }
+
+        {
+            FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest().indices("fls").fields("year");
+            FieldCapabilitiesResponse fieldCapabilitiesResponse = adminHighLevelClient.fieldCaps(fieldCapsRequest, RequestOptions.DEFAULT);
+            assertNotNull(fieldCapabilitiesResponse.get().get("year"));
+        }
+        {
+            //Though field_caps filters runtime fields out like ordinary fields
+            FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest().indices("fls").fields("year");
+            FieldCapabilitiesResponse fieldCapabilitiesResponse = highLevelClient.fieldCaps(fieldCapsRequest, RequestOptions.DEFAULT);
+            assertEquals(0, fieldCapabilitiesResponse.get().size());
+        }
+    }
+
+    private static class HighLevelClient extends RestHighLevelClient {
+        private HighLevelClient(RestClient restClient) {
+            super(restClient, (client) -> {}, Collections.emptyList());
+        }
+    }
+}