From 8d4f6929dd2f472a1e921e61db19d80fd9d9d276 Mon Sep 17 00:00:00 2001 From: Mansi Agarwal Date: Fri, 15 Nov 2019 18:15:16 -0800 Subject: [PATCH] changes for ODFE 1.3 test --- .../dlic/rest/api/AbstractApiAction.java | 4 +- .../api/OpenDistroSecurityConfigAction.java | 3 +- ...wtKeyByOpenIdConnectAuthenticatorTest.java | 17 +++ .../keybyoidc/SelfRefreshingKeySetTest.java | 10 +- .../dlic/auth/http/jwt/keybyoidc/TestJwk.java | 22 ++-- .../auth/http/jwt/keybyoidc/TestJwts.java | 16 +++ .../http/saml/HTTPSamlAuthenticatorTest.java | 55 +++++++++ .../auth/http/saml/MockSamlIdpServer.java | 9 +- .../auditlog/routing/FallbackTest.java | 2 +- .../security/auditlog/sink/KafkaSinkTest.java | 6 +- .../security/dlic/dlsfls/DlsScrollTest.java | 67 +++++++++++ .../security/dlic/dlsfls/FlsDlsTestMulti.java | 62 +++++++++++ .../dlic/dlsfls/FlsExistsFieldsTest.java | 104 ++++++++++++++++++ .../rest/api/AbstractRestApiUnitTest.java | 6 + .../security/dlic/rest/api/RolesApiTest.java | 48 +++++++- .../security/dlic/rest/api/UserApiTest.java | 33 +++++- src/test/resources/dlsfls/internal_users.yml | 3 + src/test/resources/dlsfls/roles.yml | 12 ++ src/test/resources/dlsfls/roles_mapping.yml | 3 + 19 files changed, 447 insertions(+), 35 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/DlsScrollTest.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsExistsFieldsTest.java diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractApiAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractApiAction.java index 646f2ab..cc74e3f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractApiAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractApiAction.java @@ -348,7 +348,7 @@ protected final RestChannelConsumer prepareRequest(RestRequest request, NodeClie // not 400 consumeParameters(request); - // check if SG index has been initialized + // check if .opendistro_security index has been initialized if (!ensureIndexExists()) { return channel -> internalErrorResponse(channel, ErrorType.SECURITY_NOT_INITIALIZED.getMessage()); } @@ -512,7 +512,7 @@ protected final boolean isStatic(SecurityDynamicConfiguration configuration, /** * Consume all defined parameters for the request. Before we handle the * request in subclasses where we actually need the parameter, some global - * checks are performed, e.g. check whether the SG index exists. Thus, the + * checks are performed, e.g. check whether the .security_index index exists. Thus, the * parameter(s) have not been consumed, and ES will always return a 400 with * an internal error message. * diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/OpenDistroSecurityConfigAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/OpenDistroSecurityConfigAction.java index b37331c..65963d6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/OpenDistroSecurityConfigAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/OpenDistroSecurityConfigAction.java @@ -62,7 +62,7 @@ public OpenDistroSecurityConfigAction(final Settings settings, final Path config if(allowPutOrPatch) { - //deprecated, will be removed with SG 8, use opendistro_security_config instead of sgconfig + //deprecated, will be removed with ODFE 8, use opendistro_security_config instead of config controller.registerHandler(Method.PUT, "/_opendistro/_security/api/securityconfig/{name}", this); controller.registerHandler(Method.PATCH, "/_opendistro/_security/api/securityconfig/", this); @@ -74,7 +74,6 @@ public OpenDistroSecurityConfigAction(final Settings settings, final Path config @Override protected void handleGet(RestChannel channel, RestRequest request, Client client, final JsonNode content) throws IOException{ - //final SgDynamicConfiguration configuration = load(getConfigName(), true); final SecurityDynamicConfiguration configuration = load(getConfigName(), true); filter(configuration); diff --git a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java index 9249b15..4ecc18d 100644 --- a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java +++ b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java @@ -157,4 +157,21 @@ public void testBadSignature() throws Exception { Assert.assertNull(creds); } + @Test + public void testPeculiarJsonEscaping() { + Settings settings = Settings.builder().put("openid_connect_url", mockIdpServer.getDiscoverUri()).build(); + + HTTPJwtKeyByOpenIdConnectAuthenticator jwtAuth = new HTTPJwtKeyByOpenIdConnectAuthenticator(settings, null); + + AuthCredentials creds = jwtAuth.extractCredentials( + new FakeRestRequest(ImmutableMap.of("Authorization", TestJwts.PeculiarEscaping.MC_COY_SIGNED_RSA_1), new HashMap()), + null); + + Assert.assertNotNull(creds); + Assert.assertEquals(TestJwts.MCCOY_SUBJECT, creds.getUsername()); + Assert.assertEquals(TestJwts.TEST_AUDIENCE, creds.getAttributes().get("attr.jwt.aud")); + Assert.assertEquals(0, creds.getBackendRoles().size()); + Assert.assertEquals(3, creds.getAttributes().size()); + } + } diff --git a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java index a5bb212..13f3a2e 100644 --- a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java +++ b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java @@ -35,16 +35,16 @@ public class SelfRefreshingKeySetTest { public void basicTest() throws AuthenticatorUnavailableException, BadCredentialsException { SelfRefreshingKeySet selfRefreshingKeySet = new SelfRefreshingKeySet(new MockKeySetProvider()); - JsonWebKey key1 = selfRefreshingKeySet.getKey("kid_a"); + JsonWebKey key1 = selfRefreshingKeySet.getKey("kid/a"); Assert.assertEquals(TestJwk.OCT_1_K, key1.getProperty("k")); Assert.assertEquals(1, selfRefreshingKeySet.getRefreshCount()); - JsonWebKey key2 = selfRefreshingKeySet.getKey("kid_b"); + JsonWebKey key2 = selfRefreshingKeySet.getKey("kid/b"); Assert.assertEquals(TestJwk.OCT_2_K, key2.getProperty("k")); Assert.assertEquals(1, selfRefreshingKeySet.getRefreshCount()); try { - selfRefreshingKeySet.getKey("kid_X"); + selfRefreshingKeySet.getKey("kid/X"); Assert.fail("Expected a BadCredentialsException"); } catch (BadCredentialsException e) { Assert.assertEquals(2, selfRefreshingKeySet.getRefreshCount()); @@ -62,11 +62,11 @@ public void twoThreadedTest() throws Exception { ExecutorService executorService = Executors.newCachedThreadPool(); - Future f1 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid_a")); + Future f1 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid/a")); provider.waitForCalled(); - Future f2 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid_b")); + Future f2 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid/b")); while (selfRefreshingKeySet.getQueuedGetCount() == 0) { Thread.sleep(10); diff --git a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwk.java b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwk.java index 7309aac..3b50741 100644 --- a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwk.java +++ b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwk.java @@ -30,9 +30,9 @@ class TestJwk { static final String OCT_2_K = "YP6Q3IF2qJEagV948dsicXKpG43Ci2W7ZxUpiVTBLZr1vFN9ZGUKxeXGgVWuMFYTmoHvv5AOC8BvoNOpcE3rcJNuNOqTMdujxD92CxjOykiLEKQ0Te_7xQ4LnSQjlqdIJ4U3S7qCnJLd1LxhKOGZcUhE_pjhwf7q2RUUpvC3UOyZZLog9yeflnp9nqqDy5yVqRYWZRcPI06kJTh3Z8IFi2JRJV14iUFQtOHQKuyJRMcsldKnfWl7YW3JdQ9IRN-c1lEYSEBmsavEejcqHZkbli2svqLfmCBJVWffXDRxhq0_VafiL83HC0bP9qeNKivhemw6foVmg8UMs7yJ6ao02A"; static final String OCT_3_K = "r3aeW3OK7-B4Hs3hq9BmlT1D3jRiolH9PL82XUz9xAS7dniAdmvMnN5GkOc1vqibOe2T-CC_103UglDm9D0iU9S9zn6wTuQt1L5wfZIoHd9f5IjJ_YFEzZMvsoUY_-ji_0K_ugVvBPwi9JnBQHHS4zrgmP06dGjmcnZDcIf4W_iFas3lDYSXilL1V2QhNaynpSqTarpfBGSphKv4Zg2JhsX8xB0VSaTlEq4lF8pzvpWSxXCW9CtomhB80daSuTizrmSTEPpdN3XzQ2-Tovo1ieMOfDU4csvjEk7Bwc2ThjpnA8ucKQUYpUv9joBxKuCdUltssthWnetrogjYOn_xGA"; - static final JsonWebKey OCT_1 = createOct("kid_a", "HS256", OCT_1_K); - static final JsonWebKey OCT_2 = createOct("kid_b", "HS256", OCT_2_K); - static final JsonWebKey OCT_3 = createOct("kid_c", "HS256", OCT_3_K); + static final JsonWebKey OCT_1 = createOct("kid/a", "HS256", OCT_1_K); + static final JsonWebKey OCT_2 = createOct("kid/b", "HS256", OCT_2_K); + static final JsonWebKey OCT_3 = createOct("kid/c", "HS256", OCT_3_K); static final JsonWebKey ESCAPED_SLASH_KID_OCT_1 = createOct("kid\\/_a", "HS256", OCT_1_K); static final JsonWebKey FORWARD_SLASH_KID_OCT_1 = createOct("kid/_a", "HS256", OCT_1_K); @@ -50,16 +50,16 @@ class TestJwk { static final String RSA_X_N = "jDDVUMXOXDVcaRVAT5TtuiAsLxk7XAAwyyECfmySZul7D5XVLMtGe6rP2900q3nM4BaCEiuwXjmTCZDAGlFGs2a3eQ1vbBSv9_0KGHL-gZGFPNiv0v8aR7QzZ-abhGnRy5F52PlTWsypGgG_kQpF2t2TBotvYhvVPagAt4ljllDKvY1siOvS3nh4TqcUtWcbgQZEWPmaXuhx0eLmhQJca7UEw99YlGNew48AEzt7ZnfU0Qkz3JwSz7IcPx-NfIh6BN6LwAg_ASdoM3MR8rDOtLYavmJVhutrfOpE-4-fw1mf3eLYu7xrxIplSiOIsHunTUssnTiBkXAaGqGJs604Pw"; static final String RSA_X_E = "AQAB"; - static final JsonWebKey RSA_1 = createRsa("kid_1", "RS256", RSA_1_E, RSA_1_N, RSA_1_D); - static final JsonWebKey RSA_1_PUBLIC = createRsaPublic("kid_1", "RS256", RSA_1_E, RSA_1_N); - static final JsonWebKey RSA_1_PUBLIC_NO_ALG = createRsaPublic("kid_1", null, RSA_1_E, RSA_1_N); - static final JsonWebKey RSA_1_PUBLIC_WRONG_ALG = createRsaPublic("kid_1", "HS256", RSA_1_E, RSA_1_N); + static final JsonWebKey RSA_1 = createRsa("kid/1", "RS256", RSA_1_E, RSA_1_N, RSA_1_D); + static final JsonWebKey RSA_1_PUBLIC = createRsaPublic("kid/1", "RS256", RSA_1_E, RSA_1_N); + static final JsonWebKey RSA_1_PUBLIC_NO_ALG = createRsaPublic("kid/1", null, RSA_1_E, RSA_1_N); + static final JsonWebKey RSA_1_PUBLIC_WRONG_ALG = createRsaPublic("kid/1", "HS256", RSA_1_E, RSA_1_N); - static final JsonWebKey RSA_2 = createRsa("kid_2", "RS256", RSA_2_E, RSA_2_N, RSA_2_D); - static final JsonWebKey RSA_2_PUBLIC = createRsaPublic("kid_2", "RS256", RSA_2_E, RSA_2_N); + static final JsonWebKey RSA_2 = createRsa("kid/2", "RS256", RSA_2_E, RSA_2_N, RSA_2_D); + static final JsonWebKey RSA_2_PUBLIC = createRsaPublic("kid/2", "RS256", RSA_2_E, RSA_2_N); - static final JsonWebKey RSA_X = createRsa("kid_2", "RS256", RSA_X_E, RSA_X_N, RSA_X_D); - static final JsonWebKey RSA_X_PUBLIC = createRsaPublic("kid_2", "RS256", RSA_X_E, RSA_X_N); + static final JsonWebKey RSA_X = createRsa("kid/2", "RS256", RSA_X_E, RSA_X_N, RSA_X_D); + static final JsonWebKey RSA_X_PUBLIC = createRsaPublic("kid/2", "RS256", RSA_X_E, RSA_X_N); static final JsonWebKeys RSA_1_2_PUBLIC = createJwks(RSA_1_PUBLIC, RSA_2_PUBLIC); diff --git a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwts.java b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwts.java index d60dc18..1fd0a5d 100644 --- a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwts.java +++ b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwts.java @@ -58,6 +58,10 @@ static class NoKid { static final String MC_COY_SIGNED_RSA_X = createSignedWithoutKeyId(MC_COY, TestJwk.RSA_X); } + static class PeculiarEscaping { + static final String MC_COY_SIGNED_RSA_1 = createSignedWithPeculiarEscaping(MC_COY, TestJwk.RSA_1); + } + static JwtToken create(String subject, String audience, Object... moreClaims) { JwtClaims claims = new JwtClaims(); @@ -94,4 +98,16 @@ static String createSignedWithoutKeyId(JwtToken baseJwt, JsonWebKey jwk) { return new JoseJwtProducer().processJwt(signedToken, null, JwsUtils.getSignatureProvider(jwk)); } + + static String createSignedWithPeculiarEscaping(JwtToken baseJwt, JsonWebKey jwk) { + JwsSignatureProvider signatureProvider = JwsUtils.getSignatureProvider(jwk); + JwsHeaders jwsHeaders = new JwsHeaders(); + JwtToken signedToken = new JwtToken(jwsHeaders, baseJwt.getClaims()); + + // Depends on CXF not escaping the input string. This may fail for other frameworks or versions. + jwsHeaders.setKeyId(jwk.getKeyId().replace("/", "\\/")); + + return new JoseJwtProducer().processJwt(signedToken, null, signatureProvider); + } + } diff --git a/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java b/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java index 6aee66f..ad42288 100644 --- a/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java +++ b/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java @@ -100,6 +100,7 @@ public class HTTPSamlAuthenticatorTest { @BeforeClass public static void setUp() throws Exception { mockSamlIdpServer = new MockSamlIdpServer(); + mockSamlIdpServer.start(); initSpSigningKeys(); } @@ -436,6 +437,55 @@ public void basicLogoutTestEncryptedKey() throws Exception { } + @Test + public void initialConnectionFailureTest() throws Exception { + try (MockSamlIdpServer mockSamlIdpServer = new MockSamlIdpServer()) { + + Settings settings = Settings.builder().put("idp.metadata_url", mockSamlIdpServer.getMetadataUri()) + .put("idp.min_refresh_delay", 100) + .put("kibana_url", "http://wherever").put("idp.entity_id", mockSamlIdpServer.getIdpEntityId()) + .put("exchange_key", "abc").put("roles_key", "roles").put("path.home", ".").build(); + + HTTPSamlAuthenticator samlAuthenticator = new HTTPSamlAuthenticator(settings, null); + + RestRequest restRequest = new FakeRestRequest(ImmutableMap.of(), new HashMap()); + TestRestChannel restChannel = new TestRestChannel(restRequest); + samlAuthenticator.reRequestAuthentication(restChannel, null); + + Assert.assertNull(restChannel.response); + + mockSamlIdpServer.start(); + + mockSamlIdpServer.setSignResponses(true); + mockSamlIdpServer.loadSigningKeys("saml/kirk-keystore.jks", "kirk"); + mockSamlIdpServer.setAuthenticateUser("horst"); + mockSamlIdpServer.setEndpointQueryString(null); + + Thread.sleep(500); + + AuthenticateHeaders authenticateHeaders = getAutenticateHeaders(samlAuthenticator); + + String encodedSamlResponse = mockSamlIdpServer.handleSsoGetRequestURI(authenticateHeaders.location); + + RestRequest tokenRestRequest = buildTokenExchangeRestRequest(encodedSamlResponse, authenticateHeaders); + TestRestChannel tokenRestChannel = new TestRestChannel(tokenRestRequest); + + samlAuthenticator.reRequestAuthentication(tokenRestChannel, null); + + String responseJson = new String(BytesReference.toBytes(tokenRestChannel.response.content())); + HashMap response = DefaultObjectMapper.objectMapper.readValue(responseJson, + new TypeReference>() { + }); + String authorization = (String) response.get("authorization"); + + Assert.assertNotNull("Expected authorization attribute in JSON: " + responseJson, authorization); + + JwsJwtCompactConsumer jwtConsumer = new JwsJwtCompactConsumer(authorization.replaceAll("\\s*bearer\\s*", "")); + JwtToken jwt = jwtConsumer.getJwtToken(); + + Assert.assertEquals("horst", jwt.getClaim("sub")); + } + private AuthenticateHeaders getAutenticateHeaders(HTTPSamlAuthenticator samlAuthenticator) { RestRequest restRequest = new FakeRestRequest(ImmutableMap.of(), new HashMap()); TestRestChannel restChannel = new TestRestChannel(restRequest); @@ -559,6 +609,11 @@ public void sendResponse(RestResponse response) { } + @Override + public XContentBuilder newBuilder(XContentType xContentType, XContentType responseContentType, boolean useFiltering) throws IOException { + return null; + } + } static class AuthenticateHeaders { diff --git a/src/test/java/com/amazon/dlic/auth/http/saml/MockSamlIdpServer.java b/src/test/java/com/amazon/dlic/auth/http/saml/MockSamlIdpServer.java index 83bfc61..efd6c65 100644 --- a/src/test/java/com/amazon/dlic/auth/http/saml/MockSamlIdpServer.java +++ b/src/test/java/com/amazon/dlic/auth/http/saml/MockSamlIdpServer.java @@ -181,7 +181,11 @@ class MockSamlIdpServer implements Closeable { private String defaultAssertionConsumerService; MockSamlIdpServer() throws IOException { - this(SocketUtils.findAvailableTcpPort(), false, ENTITY_ID, null); + this(SocketUtils.findAvailableTcpPort()); + } + + MockSamlIdpServer(int port) throws IOException { + this(port, false, ENTITY_ID, null); } MockSamlIdpServer(int port, boolean ssl, String idpEntityId, String endpointQueryString) throws IOException { @@ -248,6 +252,9 @@ public DefaultBHttpServerConnection createConnection(final Socket socket) throws } this.httpServer = serverBootstrap.create(); + } + + public void start() throws IOException { httpServer.start(); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/routing/FallbackTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/routing/FallbackTest.java index c825875..bd521ad 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/routing/FallbackTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/routing/FallbackTest.java @@ -126,7 +126,7 @@ private void assertLoggingSinksEmpty(AuditMessageRouter router, Category exclude List allSinks = router.categorySinks.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); allSinks = allSinks.stream().filter(sink -> (sink instanceof LoggingSink)).collect(Collectors.toList()); allSinks.removeAll(Collections.singleton(router.defaultSink)); - allSinks.remove(router.categorySinks.get(exclude)); + allSinks.removeAll(router.categorySinks.get(exclude)); for(AuditLogSink sink : allSinks) { LoggingSink loggingSink = (LoggingSink)sink; Assert.assertEquals(0, loggingSink.messages.size()); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/sink/KafkaSinkTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/sink/KafkaSinkTest.java index 498ff74..fa6e0bf 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/sink/KafkaSinkTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/sink/KafkaSinkTest.java @@ -40,7 +40,7 @@ public class KafkaSinkTest extends AbstractAuditlogiUnitTest { @ClassRule - public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, 1, "compliance"); + public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 1, "compliance"); @Test public void testKafka() throws Exception { @@ -57,7 +57,7 @@ public void testKafka() throws Exception { Assert.assertEquals(KafkaSink.class, sink.getClass()); boolean success = sink.doStore(MockAuditMessageFactory.validAuditMessage(Category.MISSING_PRIVILEGES)); Assert.assertTrue(success); - ConsumerRecords records = consumer.poll(10000); + ConsumerRecords records = consumer.poll(Duration.ofSeconds(10)); Assert.assertEquals(1, records.count()); } finally { sink.close(); @@ -68,7 +68,7 @@ public void testKafka() throws Exception { private KafkaConsumer createConsumer() { Properties props = new Properties(); - props.put("bootstrap.servers", embeddedKafka.getBrokersAsString()); + props.put("bootstrap.servers", embeddedKafka.getEmbeddedKafka().getBrokersAsString()); props.put("auto.offset.reset", "earliest"); props.put("group.id", "mygroup"+System.currentTimeMillis()+"_"+new Random().nextDouble()); props.put("key.deserializer", "org.apache.kafka.common.serialization.LongDeserializer"); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/DlsScrollTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/DlsScrollTest.java new file mode 100644 index 0000000..fff93ce --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/DlsScrollTest.java @@ -0,0 +1,67 @@ +package com.amazon.opendistroforelasticsearch.security.dlic.dlsfls; + +import org.apache.http.HttpStatus; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.junit.Assert; +import org.junit.Test; + +import com.floragunn.searchguard.test.helper.file.FileHelper; +import com.floragunn.searchguard.test.helper.rest.RestHelper.HttpResponse; + +public class DlsScrollTest extends AbstractDlsFlsTest{ + + + @Override + protected void populateData(TransportClient tc) { + + tc.index(new IndexRequest("deals").type("deals").id("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source("{\"amount\": 3}", XContentType.JSON)).actionGet(); //not in + + tc.index(new IndexRequest("deals").type("deals").id("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source("{\"amount\": 10}", XContentType.JSON)).actionGet(); //not in + + tc.index(new IndexRequest("deals").type("deals").id("2").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source("{\"amount\": 1500}", XContentType.JSON)).actionGet(); + + tc.index(new IndexRequest("deals").type("deals").id("4").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source("{\"amount\": 21500}", XContentType.JSON)).actionGet(); //not in + + for(int i=0; i<100; i++) { + tc.index(new IndexRequest("deals").type("deals").id("gen"+i).setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .source("{\"amount\": 1500}", XContentType.JSON)).actionGet(); + } + } + + + @Test + public void testDlsScroll() throws Exception { + + setup(); + + HttpResponse res; + Assert.assertEquals(HttpStatus.SC_OK, (res=rh.executeGetRequest("/deals/_search?scroll=1m&pretty=true&size=5", encodeBasicHeader("dept_manager", "password"))).getStatusCode()); + Assert.assertTrue(res.getBody().contains("\"value\" : 101,")); + + int c=0; + + while(true) { + int start = res.getBody().indexOf("_scroll_id") + 15; + String scrollid = res.getBody().substring(start, res.getBody().indexOf("\"", start+1)); + Assert.assertEquals(HttpStatus.SC_OK, (res=rh.executePostRequest("/_search/scroll?pretty=true", "{\"scroll\" : \"1m\", \"scroll_id\" : \""+scrollid+"\"}", encodeBasicHeader("dept_manager", "password"))).getStatusCode()); + Assert.assertTrue(res.getBody().contains("\"value\" : 101,")); + Assert.assertFalse(res.getBody().contains("\"amount\" : 3")); + Assert.assertFalse(res.getBody().contains("\"amount\" : 10")); + Assert.assertFalse(res.getBody().contains("\"amount\" : 21500")); + c++; + + if(res.getBody().contains("\"hits\" : [ ]")) { + break; + } + } + + Assert.assertEquals(21, c); + } +} \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsDlsTestMulti.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsDlsTestMulti.java index dcbebf5..cb0c1aa 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsDlsTestMulti.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsDlsTestMulti.java @@ -208,4 +208,66 @@ public void testDlsFls() throws Exception { Assert.assertTrue(res.getBody().contains("\"amount\" : 20")); Assert.assertTrue(res.getBody().contains("\"found\" : false")); } + + @Test + public void testDlsSuggest() throws Exception { + + setup(); + + HttpResponse res; + String query = + + "{"+ + "\"query\": {"+ + "\"range\" : {"+ + "\"amount\" : {"+ + "\"gte\" : 11,"+ + "\"lte\" : 50000,"+ + "\"boost\" : 1.0"+ + "}"+ + "}"+ + "},"+ + "\"suggest\" : {\n" + + " \"thesuggestion\" : {\n" + + " \"text\" : \"cust\",\n" + + " \"term\" : {\n" + + " \"field\" : \"customer.name\"\n" + + " }\n" + + " }\n" + + " }"+ + "}"; + + Assert.assertEquals(HttpStatus.SC_OK, (res = rh.executePostRequest("/deals/_search?pretty", query, encodeBasicHeader("admin", "admin"))).getStatusCode()); + Assert.assertTrue(res.getBody().contains("thesuggestion")); + + Assert.assertEquals(HttpStatus.SC_OK, (res = rh.executePostRequest("/deals/_search?pretty", query, encodeBasicHeader("dept_manager_multi", "password"))).getStatusCode()); + Assert.assertTrue(res.getBody().contains("thesuggestion")); + } + + @Test + public void testDlsSuggestOnly() throws Exception { + + setup(); + + HttpResponse res; + String query = + + "{"+ + "\"suggest\" : {\n" + + " \"thesuggestion\" : {\n" + + " \"text\" : \"cust\",\n" + + " \"term\" : {\n" + + " \"field\" : \"customer.name\"\n" + + " }\n" + + " }\n" + + " }"+ + "}"; + + Assert.assertEquals(HttpStatus.SC_OK, (res = rh.executePostRequest("/deals/_search?pretty", query, encodeBasicHeader("admin", "admin"))).getStatusCode()); + Assert.assertTrue(res.getBody().contains("thesuggestion")); + + Assert.assertEquals(HttpStatus.SC_OK, (res = rh.executePostRequest("/deals/_search?pretty", query, encodeBasicHeader("dept_manager_multi", "password"))).getStatusCode()); + Assert.assertTrue(res.getBody().contains("thesuggestion")); + } + } \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsExistsFieldsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsExistsFieldsTest.java new file mode 100644 index 0000000..fc16641 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/FlsExistsFieldsTest.java @@ -0,0 +1,104 @@ +package com.amazon.opendistroforelasticsearch.security.dlic.dlsfls; + +import org.apache.http.HttpStatus; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.junit.Assert; +import org.junit.Test; + +import com.amazon.opendistroforelasticsearch.security.test.helper.rest.RestHelper.HttpResponse; + +public class FlsExistsFieldsTest extends AbstractDlsFlsTest { + + protected void populateData(TransportClient tc) { + + tc.admin().indices().create(new CreateIndexRequest("data").mapping("doc", + "@timestamp", "type=date", + "host", "type=text,norms=false", + "response", "type=text,norms=false", + "non-existing", "type=text,norms=false" + )) + .actionGet(); + + for (int i = 0; i < 1; i++) { + String doc = "{\"host\" : \"myhost"+i+"\",\n" + + " \"@timestamp\" : \"2018-01-18T09:03:25.877Z\",\n" + + " \"response\": \"404\"}"; + tc.index(new IndexRequest("data").type("doc").id("a-normal-" + i).setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(doc, + XContentType.JSON)).actionGet(); + } + + for (int i = 0; i < 1; i++) { + String doc = "{" + + " \"@timestamp\" : \"2017-01-18T09:03:25.877Z\",\n" + + " \"response\": \"200\"}"; + tc.index(new IndexRequest("data").type("doc").id("b-missing1-" + i).setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(doc, + XContentType.JSON)).actionGet(); + } + + for (int i = 0; i < 1; i++) { + String doc = "{\"host\" : \"myhost"+i+"\",\n" + + " \"@timestamp\" : \"2018-01-18T09:03:25.877Z\",\n" + + " \"non-existing\": \"xxx\","+ + " \"response\": \"403\"}"; + tc.index(new IndexRequest("data").type("doc").id("c-missing2-" + i).setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(doc, + XContentType.JSON)).actionGet(); + } + + } + + @Test + public void testExistsField() throws Exception { + setup(); + + String query = "{\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + + " \"must_not\": \n" + + " {\n" + + " \"exists\": {\n" + + " \"field\": \"non-existing\"\n" + + " \n" + + " }\n" + + " },\n" + + + " \"must\": [\n" + + " {\n" + + " \"exists\": {\n" + + " \"field\": \"@timestamp\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"exists\": {\n" + + " \"field\": \"host\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + + HttpResponse res; + Assert.assertEquals(HttpStatus.SC_OK, + (res = rh.executePostRequest("/data/_search?pretty", query, encodeBasicHeader("admin", "admin"))).getStatusCode()); + System.out.println(res.getBody()); + Assert.assertTrue(res.getBody().contains("\"value\" : 1,\n \"relation")); + Assert.assertTrue(res.getBody().contains("a-normal-0")); + Assert.assertTrue(res.getBody().contains("response")); + Assert.assertTrue(res.getBody().contains("404")); + + //only see's - timestamp and host field + //therefore non-existing does not exist so we expect c-missing2-0 to be returned + Assert.assertEquals(HttpStatus.SC_OK, + (res = rh.executePostRequest("/data/_search?pretty", query, encodeBasicHeader("fls_exists", "password"))).getStatusCode()); + System.out.println(res.getBody()); + Assert.assertTrue(res.getBody().contains("\"value\" : 2,\n \"relation")); + Assert.assertTrue(res.getBody().contains("a-normal-0")); + Assert.assertTrue(res.getBody().contains("c-missing2-0")); + Assert.assertFalse(res.getBody().contains("response")); + } +} \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractRestApiUnitTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractRestApiUnitTest.java index 63629de..bf6458f 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractRestApiUnitTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractRestApiUnitTest.java @@ -218,6 +218,12 @@ protected void setupStarfleetIndex() throws Exception { rh.sendHTTPClientCertificate = sendHTTPClientCertificate; } + protected void assertHealthy() throws Exception { + Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("_opendistro/_security/health?pretty").getStatusCode()); + Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("_opendistro/_security/authinfo?pretty", encodeBasicHeader("admin", "admin")).getStatusCode()); + Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("*/_search?pretty", encodeBasicHeader("admin", "admin")).getStatusCode()); + } + protected Settings defaultNodeSettings(boolean enableRestSSL) { Settings.Builder builder = Settings.builder(); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/RolesApiTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/RolesApiTest.java index 1c934d4..7c5f3f7 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/RolesApiTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/RolesApiTest.java @@ -61,11 +61,50 @@ public void testAllRolesNotContainMetaHeader() throws Exception { rh.keystore = "restapi/kirk-keystore.jks"; rh.sendHTTPClientCertificate = true; - HttpResponse response = rh.executeGetRequest("/_opendistro/_security/api/roles"); + HttpResponse response = rh.executeGetRequest("_opendistro/_security/api/roles"); Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); Assert.assertFalse(response.getBody().contains("_meta")); } + @Test + public void testPutDuplicateKeys() throws Exception { + + setup(); + + rh.keystore = "restapi/kirk-keystore.jks"; + rh.sendHTTPClientCertificate = true; + HttpResponse response = rh.executePutRequest("_opendistro/_security/api/roles/dup", "{ \"cluster_permissions\": [\"*\"], \"cluster_permissions\": [\"*\"] }"); + Assert.assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); + Assert.assertTrue(response.getBody().contains("JsonParseException")); + assertHealthy(); + } + + @Test + public void testPutUnknownKey() throws Exception { + + setup(); + + rh.keystore = "restapi/kirk-keystore.jks"; + rh.sendHTTPClientCertificate = true; + HttpResponse response = rh.executePutRequest("_opendistro/_security/api/roles/dup", "{ \"unknownkey\": [\"*\"], \"cluster_permissions\": [\"*\"] }"); + Assert.assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); + Assert.assertTrue(response.getBody().contains("invalid_keys")); + assertHealthy(); + } + + @Test + public void testPutInvalidJson() throws Exception { + + setup(); + + rh.keystore = "restapi/kirk-keystore.jks"; + rh.sendHTTPClientCertificate = true; + HttpResponse response = rh.executePutRequest("_searchguard/api/roles/dup", "{ \"invalid\"::{{ [\"*\"], \"cluster_permissions\": [\"*\"] }"); + Assert.assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); + Assert.assertTrue(response.getBody().contains("JsonParseException")); + assertHealthy(); + } + @Test public void testRolesApi() throws Exception { @@ -90,17 +129,14 @@ public void testRolesApi() throws Exception { response = rh.executeGetRequest("/_opendistro/_security/api/roles/nothinghthere", new Header[0]); Assert.assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatusCode()); - // GET, new URL endpoint in SG6 response = rh.executeGetRequest("/_opendistro/_security/api/roles/", new Header[0]); Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - // GET, new URL endpoint in SG6 response = rh.executeGetRequest("/_opendistro/_security/api/roles", new Header[0]); Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); Assert.assertTrue(response.getBody().contains("\"cluster_permissions\":[\"*\"]")); Assert.assertFalse(response.getBody().contains("\"cluster_permissions\" : [")); - // GET, new URL endpoint in SG6, pretty response = rh.executeGetRequest("/_opendistro/_security/api/roles?pretty", new Header[0]); Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); Assert.assertFalse(response.getBody().contains("\"cluster_permissions\":[\"*\"]")); @@ -147,7 +183,7 @@ public void testRolesApi() throws Exception { // user has only role starfleet left, role has READ access only checkWriteAccess(HttpStatus.SC_FORBIDDEN, "picard", "picard", "sf", "ships", 1); - // ES7 only supports one doc type, but SG permission checks run first + // ES7 only supports one doc type, but Opendistro permission checks run first // So we also get a 403 FORBIDDEN when tring to add new document type checkWriteAccess(HttpStatus.SC_FORBIDDEN, "picard", "picard", "sf", "public", 0); @@ -208,7 +244,7 @@ public void testRolesApi() throws Exception { checkReadAccess(HttpStatus.SC_OK, "picard", "picard", "sf", "ships", 0); // now picard is only in opendistro_security_role_starfleet, which has write access to - // all indices. We collapse all document types in SG7 so this permission in the + // all indices. We collapse all document types in ODFE7 so this permission in the // starfleet role grants all permissions: // public: // - 'indices:*' diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/UserApiTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/UserApiTest.java index 2a4ee91..edd5466 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/UserApiTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/UserApiTest.java @@ -34,6 +34,31 @@ public class UserApiTest extends AbstractRestApiUnitTest { + @Test + public void testOpenDistroSecurityRoles() throws Exception { + + setup(); + + rh.keystore = "restapi/kirk-keystore.jks"; + rh.sendHTTPClientCertificate = true; + + // initial configuration, 5 users + HttpResponse response = rh + .executeGetRequest("_opendistro/_security/api/" + CType.INTERNALUSERS.toLCString()); + Assert.assertEquals(response.getBody(), HttpStatus.SC_OK, response.getStatusCode()); + Settings settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); + Assert.assertEquals(35, settings.size()); + + response = rh.executePatchRequest("/_opendistro/_security/api/internalusers", "[{ \"op\": \"add\", \"path\": \"/newuser\", \"value\": {\"password\": \"newuser\", \"opendistro_security_roles\": [\"opendistro_security_all_access\"] } }]", new Header[0]); + Assert.assertEquals(response.getBody(), HttpStatus.SC_OK, response.getStatusCode()); + + response = rh.executeGetRequest("/_opendistro/_security/api/internalusers/newuser", new Header[0]); + Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); + Assert.assertTrue(response.getBody().contains("\"opendistro_security_roles\":[\"all_access\"]")); + + checkGeneralAccess(HttpStatus.SC_OK, "newuser", "newuser"); + } + @Test public void testUserApi() throws Exception { @@ -47,7 +72,7 @@ public void testUserApi() throws Exception { .executeGetRequest("_opendistro/_security/api/" + CType.INTERNALUSERS.toLCString()); Assert.assertEquals(response.getBody(), HttpStatus.SC_OK, response.getStatusCode()); Settings settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(30, settings.size()); + Assert.assertEquals(35, settings.size()); // --- GET // GET, user admin, exists @@ -55,7 +80,7 @@ public void testUserApi() throws Exception { Assert.assertEquals(response.getBody(), HttpStatus.SC_OK, response.getStatusCode()); System.out.println(response.getBody()); settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(6, settings.size()); + Assert.assertEquals(7, settings.size()); // hash must be filtered Assert.assertEquals("", settings.get("admin.hash")); @@ -345,7 +370,7 @@ public void testPasswordRules() throws Exception { Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); System.out.println(response.getBody()); Settings settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(30, settings.size()); + Assert.assertEquals(35, settings.size()); addUserWithPassword("tooshoort", "123", HttpStatus.SC_BAD_REQUEST); addUserWithPassword("tooshoort", "1234567", HttpStatus.SC_BAD_REQUEST); @@ -408,7 +433,7 @@ public void testUserApiWithDots() throws Exception { .executeGetRequest("_opendistro/_security/api/" + CType.INTERNALUSERS.toLCString()); Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); Settings settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(30, settings.size()); + Assert.assertEquals(35, settings.size()); addUserWithPassword(".my.dotuser0", "$2a$12$n5nubfWATfQjSYHiWtUyeOxMIxFInUHOAx8VMmGmxFNPGpaBmeB.m", HttpStatus.SC_CREATED); diff --git a/src/test/resources/dlsfls/internal_users.yml b/src/test/resources/dlsfls/internal_users.yml index 30d19c4..33c85a4 100644 --- a/src/test/resources/dlsfls/internal_users.yml +++ b/src/test/resources/dlsfls/internal_users.yml @@ -168,3 +168,6 @@ user_masked_custom: date_math: #password hash: $2a$12$YCBrpxYyFusK609FurY5Ee3BlmuzWw0qHwpwqEyNhM2.XnQY3Bxpe +fls_exists: + #password + hash: $2a$12$YCBrpxYyFusK609FurY5Ee3BlmuzWw0qHwpwqEyNhM2.XnQY3Bxpe diff --git a/src/test/resources/dlsfls/roles.yml b/src/test/resources/dlsfls/roles.yml index a6e81dd..d9b7e76 100644 --- a/src/test/resources/dlsfls/roles.yml +++ b/src/test/resources/dlsfls/roles.yml @@ -2,6 +2,18 @@ _meta: type: "roles" config_version: 2 +opendistro_security_fls_exists: + cluster_permissions: + - "*" + index_permissions: + - index_patterns: + - "data" + allowed_actions: + - "*" + fls: + - "@timestamp" + - "host" + #- "non-existing" opendistro_security_date_math: index_permissions: - index_patterns: diff --git a/src/test/resources/dlsfls/roles_mapping.yml b/src/test/resources/dlsfls/roles_mapping.yml index 694a532..ebe633d 100644 --- a/src/test/resources/dlsfls/roles_mapping.yml +++ b/src/test/resources/dlsfls/roles_mapping.yml @@ -231,3 +231,6 @@ opendistro_security_combined: opendistro_security_date_math: users: - date_math +opendistro_security_fls_exists: + users: + - fls_exists