changes for ODFE 1.3 test

src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/AbstractApiAction.java

@@ -348,7 +348,7 @@ protected final RestChannelConsumer prepareRequest(RestRequest request, NodeClie
 		// not 400
 		consumeParameters(request);
 
-		// check if SG index has been initialized
+		// check if .opendistro_security index has been initialized
 		if (!ensureIndexExists()) {
 			return channel -> internalErrorResponse(channel, ErrorType.SECURITY_NOT_INITIALIZED.getMessage());
 		}
@@ -512,7 +512,7 @@ protected final boolean isStatic(SecurityDynamicConfiguration<?> configuration,
 	/**
 	 * Consume all defined parameters for the request. Before we handle the
 	 * request in subclasses where we actually need the parameter, some global
-	 * checks are performed, e.g. check whether the SG index exists. Thus, the
+	 * checks are performed, e.g. check whether the .security_index index exists. Thus, the
 	 * parameter(s) have not been consumed, and ES will always return a 400 with
 	 * an internal error message.
 	 *

src/main/java/com/amazon/opendistroforelasticsearch/security/dlic/rest/api/OpenDistroSecurityConfigAction.java

@@ -62,7 +62,7 @@ public OpenDistroSecurityConfigAction(final Settings settings, final Path config
 
         if(allowPutOrPatch) {
 
-            //deprecated, will be removed with SG 8, use opendistro_security_config instead of sgconfig
+            //deprecated, will be removed with ODFE 8, use opendistro_security_config instead of config
             controller.registerHandler(Method.PUT, "/_opendistro/_security/api/securityconfig/{name}", this);
             controller.registerHandler(Method.PATCH, "/_opendistro/_security/api/securityconfig/", this);
 
@@ -74,7 +74,6 @@ public OpenDistroSecurityConfigAction(final Settings settings, final Path config
 
     @Override
     protected void handleGet(RestChannel channel, RestRequest request, Client client, final JsonNode content) throws IOException{
-        //final SgDynamicConfiguration<?> configuration = load(getConfigName(), true);
         final SecurityDynamicConfiguration<?> configuration = load(getConfigName(), true);
 
         filter(configuration);

src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/HTTPJwtKeyByOpenIdConnectAuthenticatorTest.java

@@ -157,4 +157,21 @@ public void testBadSignature() throws Exception {
 		Assert.assertNull(creds);
 	}
 
+	@Test
+	public void testPeculiarJsonEscaping() {
+		Settings settings = Settings.builder().put("openid_connect_url", mockIdpServer.getDiscoverUri()).build();
+
+		HTTPJwtKeyByOpenIdConnectAuthenticator jwtAuth = new HTTPJwtKeyByOpenIdConnectAuthenticator(settings, null);
+
+		AuthCredentials creds = jwtAuth.extractCredentials(
+				new FakeRestRequest(ImmutableMap.of("Authorization", TestJwts.PeculiarEscaping.MC_COY_SIGNED_RSA_1), new HashMap<String, String>()),
+				null);
+
+		Assert.assertNotNull(creds);
+		Assert.assertEquals(TestJwts.MCCOY_SUBJECT, creds.getUsername());
+		Assert.assertEquals(TestJwts.TEST_AUDIENCE, creds.getAttributes().get("attr.jwt.aud"));
+		Assert.assertEquals(0, creds.getBackendRoles().size());
+		Assert.assertEquals(3, creds.getAttributes().size());
+	}
+
 }

src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java

@@ -35,16 +35,16 @@ public class SelfRefreshingKeySetTest {
 	public void basicTest() throws AuthenticatorUnavailableException, BadCredentialsException {
 		SelfRefreshingKeySet selfRefreshingKeySet = new SelfRefreshingKeySet(new MockKeySetProvider());
 
-		JsonWebKey key1 = selfRefreshingKeySet.getKey("kid_a");
+		JsonWebKey key1 = selfRefreshingKeySet.getKey("kid/a");
 		Assert.assertEquals(TestJwk.OCT_1_K, key1.getProperty("k"));
 		Assert.assertEquals(1, selfRefreshingKeySet.getRefreshCount());
 
-		JsonWebKey key2 = selfRefreshingKeySet.getKey("kid_b");
+		JsonWebKey key2 = selfRefreshingKeySet.getKey("kid/b");
 		Assert.assertEquals(TestJwk.OCT_2_K, key2.getProperty("k"));
 		Assert.assertEquals(1, selfRefreshingKeySet.getRefreshCount());
 
 		try {
-			selfRefreshingKeySet.getKey("kid_X");
+			selfRefreshingKeySet.getKey("kid/X");
 			Assert.fail("Expected a BadCredentialsException");
 		} catch (BadCredentialsException e) {
 			Assert.assertEquals(2, selfRefreshingKeySet.getRefreshCount());
@@ -62,11 +62,11 @@ public void twoThreadedTest() throws Exception {
 
 		ExecutorService executorService = Executors.newCachedThreadPool();
 
-		Future<JsonWebKey> f1 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid_a"));
+		Future<JsonWebKey> f1 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid/a"));
 
 		provider.waitForCalled();
 
-		Future<JsonWebKey> f2 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid_b"));
+		Future<JsonWebKey> f2 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid/b"));
 
 		while (selfRefreshingKeySet.getQueuedGetCount() == 0) {
 			Thread.sleep(10);

src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwk.java

@@ -30,9 +30,9 @@ class TestJwk {
 	static final String OCT_2_K = "YP6Q3IF2qJEagV948dsicXKpG43Ci2W7ZxUpiVTBLZr1vFN9ZGUKxeXGgVWuMFYTmoHvv5AOC8BvoNOpcE3rcJNuNOqTMdujxD92CxjOykiLEKQ0Te_7xQ4LnSQjlqdIJ4U3S7qCnJLd1LxhKOGZcUhE_pjhwf7q2RUUpvC3UOyZZLog9yeflnp9nqqDy5yVqRYWZRcPI06kJTh3Z8IFi2JRJV14iUFQtOHQKuyJRMcsldKnfWl7YW3JdQ9IRN-c1lEYSEBmsavEejcqHZkbli2svqLfmCBJVWffXDRxhq0_VafiL83HC0bP9qeNKivhemw6foVmg8UMs7yJ6ao02A";
 	static final String OCT_3_K = "r3aeW3OK7-B4Hs3hq9BmlT1D3jRiolH9PL82XUz9xAS7dniAdmvMnN5GkOc1vqibOe2T-CC_103UglDm9D0iU9S9zn6wTuQt1L5wfZIoHd9f5IjJ_YFEzZMvsoUY_-ji_0K_ugVvBPwi9JnBQHHS4zrgmP06dGjmcnZDcIf4W_iFas3lDYSXilL1V2QhNaynpSqTarpfBGSphKv4Zg2JhsX8xB0VSaTlEq4lF8pzvpWSxXCW9CtomhB80daSuTizrmSTEPpdN3XzQ2-Tovo1ieMOfDU4csvjEk7Bwc2ThjpnA8ucKQUYpUv9joBxKuCdUltssthWnetrogjYOn_xGA";
 
-	static final JsonWebKey OCT_1 = createOct("kid_a", "HS256", OCT_1_K);
-	static final JsonWebKey OCT_2 = createOct("kid_b", "HS256", OCT_2_K);
-	static final JsonWebKey OCT_3 = createOct("kid_c", "HS256", OCT_3_K);
+	static final JsonWebKey OCT_1 = createOct("kid/a", "HS256", OCT_1_K);
+	static final JsonWebKey OCT_2 = createOct("kid/b", "HS256", OCT_2_K);
+	static final JsonWebKey OCT_3 = createOct("kid/c", "HS256", OCT_3_K);
 	static final JsonWebKey ESCAPED_SLASH_KID_OCT_1 = createOct("kid\\/_a", "HS256", OCT_1_K);
 	static final JsonWebKey FORWARD_SLASH_KID_OCT_1 = createOct("kid/_a", "HS256", OCT_1_K);
 
@@ -50,16 +50,16 @@ class TestJwk {
 	static final String RSA_X_N = "jDDVUMXOXDVcaRVAT5TtuiAsLxk7XAAwyyECfmySZul7D5XVLMtGe6rP2900q3nM4BaCEiuwXjmTCZDAGlFGs2a3eQ1vbBSv9_0KGHL-gZGFPNiv0v8aR7QzZ-abhGnRy5F52PlTWsypGgG_kQpF2t2TBotvYhvVPagAt4ljllDKvY1siOvS3nh4TqcUtWcbgQZEWPmaXuhx0eLmhQJca7UEw99YlGNew48AEzt7ZnfU0Qkz3JwSz7IcPx-NfIh6BN6LwAg_ASdoM3MR8rDOtLYavmJVhutrfOpE-4-fw1mf3eLYu7xrxIplSiOIsHunTUssnTiBkXAaGqGJs604Pw";
 	static final String RSA_X_E = "AQAB";
 
-	static final JsonWebKey RSA_1 = createRsa("kid_1", "RS256", RSA_1_E, RSA_1_N, RSA_1_D);
-	static final JsonWebKey RSA_1_PUBLIC = createRsaPublic("kid_1", "RS256", RSA_1_E, RSA_1_N);
-	static final JsonWebKey RSA_1_PUBLIC_NO_ALG = createRsaPublic("kid_1", null, RSA_1_E, RSA_1_N);
-    static final JsonWebKey RSA_1_PUBLIC_WRONG_ALG = createRsaPublic("kid_1", "HS256", RSA_1_E, RSA_1_N);
+	static final JsonWebKey RSA_1 = createRsa("kid/1", "RS256", RSA_1_E, RSA_1_N, RSA_1_D);
+	static final JsonWebKey RSA_1_PUBLIC = createRsaPublic("kid/1", "RS256", RSA_1_E, RSA_1_N);
+	static final JsonWebKey RSA_1_PUBLIC_NO_ALG = createRsaPublic("kid/1", null, RSA_1_E, RSA_1_N);
+    static final JsonWebKey RSA_1_PUBLIC_WRONG_ALG = createRsaPublic("kid/1", "HS256", RSA_1_E, RSA_1_N);
 
-	static final JsonWebKey RSA_2 = createRsa("kid_2", "RS256", RSA_2_E, RSA_2_N, RSA_2_D);
-	static final JsonWebKey RSA_2_PUBLIC = createRsaPublic("kid_2", "RS256", RSA_2_E, RSA_2_N);
+	static final JsonWebKey RSA_2 = createRsa("kid/2", "RS256", RSA_2_E, RSA_2_N, RSA_2_D);
+	static final JsonWebKey RSA_2_PUBLIC = createRsaPublic("kid/2", "RS256", RSA_2_E, RSA_2_N);
 
-	static final JsonWebKey RSA_X = createRsa("kid_2", "RS256", RSA_X_E, RSA_X_N, RSA_X_D);
-	static final JsonWebKey RSA_X_PUBLIC = createRsaPublic("kid_2", "RS256", RSA_X_E, RSA_X_N);
+	static final JsonWebKey RSA_X = createRsa("kid/2", "RS256", RSA_X_E, RSA_X_N, RSA_X_D);
+	static final JsonWebKey RSA_X_PUBLIC = createRsaPublic("kid/2", "RS256", RSA_X_E, RSA_X_N);
 
 	static final JsonWebKeys RSA_1_2_PUBLIC = createJwks(RSA_1_PUBLIC, RSA_2_PUBLIC);
 

src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/TestJwts.java

@@ -58,6 +58,10 @@ static class NoKid {
 		static final String MC_COY_SIGNED_RSA_X = createSignedWithoutKeyId(MC_COY, TestJwk.RSA_X);
 	}
 
+	static class PeculiarEscaping {
+		static final String MC_COY_SIGNED_RSA_1 = createSignedWithPeculiarEscaping(MC_COY, TestJwk.RSA_1);
+	}
+
 	static JwtToken create(String subject, String audience, Object... moreClaims) {
 		JwtClaims claims = new JwtClaims();
 
@@ -94,4 +98,16 @@ static String createSignedWithoutKeyId(JwtToken baseJwt, JsonWebKey jwk) {
 
 		return new JoseJwtProducer().processJwt(signedToken, null, JwsUtils.getSignatureProvider(jwk));
 	}
+
+	static String createSignedWithPeculiarEscaping(JwtToken baseJwt, JsonWebKey jwk) {
+		JwsSignatureProvider signatureProvider = JwsUtils.getSignatureProvider(jwk);
+		JwsHeaders jwsHeaders = new JwsHeaders();
+		JwtToken signedToken = new JwtToken(jwsHeaders, baseJwt.getClaims());
+
+		// Depends on CXF not escaping the input string. This may fail for other frameworks or versions.
+		jwsHeaders.setKeyId(jwk.getKeyId().replace("/", "\\/"));
+
+		return new JoseJwtProducer().processJwt(signedToken, null, signatureProvider);
+	}
+
 }

src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java

@@ -100,6 +100,7 @@ public class HTTPSamlAuthenticatorTest {
     @BeforeClass
     public static void setUp() throws Exception {
         mockSamlIdpServer = new MockSamlIdpServer();
+        mockSamlIdpServer.start();
         initSpSigningKeys();
     }
 
@@ -436,6 +437,55 @@ public void basicLogoutTestEncryptedKey() throws Exception {
 
     }
 
+    @Test
+    public void initialConnectionFailureTest() throws Exception {
+        try (MockSamlIdpServer mockSamlIdpServer = new MockSamlIdpServer()) {
+
+            Settings settings = Settings.builder().put("idp.metadata_url", mockSamlIdpServer.getMetadataUri())
+                    .put("idp.min_refresh_delay", 100)
+                    .put("kibana_url", "http://wherever").put("idp.entity_id", mockSamlIdpServer.getIdpEntityId())
+                    .put("exchange_key", "abc").put("roles_key", "roles").put("path.home", ".").build();
+
+            HTTPSamlAuthenticator samlAuthenticator = new HTTPSamlAuthenticator(settings, null);
+
+            RestRequest restRequest = new FakeRestRequest(ImmutableMap.of(), new HashMap<String, String>());
+            TestRestChannel restChannel = new TestRestChannel(restRequest);
+            samlAuthenticator.reRequestAuthentication(restChannel, null);
+
+            Assert.assertNull(restChannel.response);
+
+            mockSamlIdpServer.start();
+
+            mockSamlIdpServer.setSignResponses(true);
+            mockSamlIdpServer.loadSigningKeys("saml/kirk-keystore.jks", "kirk");
+            mockSamlIdpServer.setAuthenticateUser("horst");
+            mockSamlIdpServer.setEndpointQueryString(null);
+
+            Thread.sleep(500);
+
+            AuthenticateHeaders authenticateHeaders = getAutenticateHeaders(samlAuthenticator);
+
+            String encodedSamlResponse = mockSamlIdpServer.handleSsoGetRequestURI(authenticateHeaders.location);
+
+            RestRequest tokenRestRequest = buildTokenExchangeRestRequest(encodedSamlResponse, authenticateHeaders);
+            TestRestChannel tokenRestChannel = new TestRestChannel(tokenRestRequest);
+
+            samlAuthenticator.reRequestAuthentication(tokenRestChannel, null);
+
+            String responseJson = new String(BytesReference.toBytes(tokenRestChannel.response.content()));
+            HashMap<String, Object> response = DefaultObjectMapper.objectMapper.readValue(responseJson,
+                    new TypeReference<HashMap<String, Object>>() {
+                    });
+            String authorization = (String) response.get("authorization");
+
+            Assert.assertNotNull("Expected authorization attribute in JSON: " + responseJson, authorization);
+
+            JwsJwtCompactConsumer jwtConsumer = new JwsJwtCompactConsumer(authorization.replaceAll("\\s*bearer\\s*", ""));
+            JwtToken jwt = jwtConsumer.getJwtToken();
+
+            Assert.assertEquals("horst", jwt.getClaim("sub"));
+        }
+
     private AuthenticateHeaders getAutenticateHeaders(HTTPSamlAuthenticator samlAuthenticator) {
         RestRequest restRequest = new FakeRestRequest(ImmutableMap.of(), new HashMap<String, String>());
         TestRestChannel restChannel = new TestRestChannel(restRequest);
@@ -559,6 +609,11 @@ public void sendResponse(RestResponse response) {
 
         }
 
+        @Override
+        public XContentBuilder newBuilder(XContentType xContentType, XContentType responseContentType, boolean useFiltering) throws IOException {
+            return null;
+        }
+
     }
 
     static class AuthenticateHeaders {

src/test/java/com/amazon/dlic/auth/http/saml/MockSamlIdpServer.java

@@ -181,7 +181,11 @@ class MockSamlIdpServer implements Closeable {
     private String defaultAssertionConsumerService;
 
     MockSamlIdpServer() throws IOException {
-        this(SocketUtils.findAvailableTcpPort(), false, ENTITY_ID, null);
+        this(SocketUtils.findAvailableTcpPort());
+    }
+
+    MockSamlIdpServer(int port) throws IOException {
+        this(port, false, ENTITY_ID, null);
     }
 
     MockSamlIdpServer(int port, boolean ssl, String idpEntityId, String endpointQueryString) throws IOException {
@@ -248,6 +252,9 @@ public DefaultBHttpServerConnection createConnection(final Socket socket) throws
         }
 
         this.httpServer = serverBootstrap.create();
+    }
+
+    public void start() throws IOException {
 
         httpServer.start();
 

src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/routing/FallbackTest.java

@@ -126,7 +126,7 @@ private void assertLoggingSinksEmpty(AuditMessageRouter router, Category exclude
 		List<AuditLogSink> allSinks = router.categorySinks.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
 		allSinks = allSinks.stream().filter(sink -> (sink instanceof LoggingSink)).collect(Collectors.toList());
 		allSinks.removeAll(Collections.singleton(router.defaultSink));
-		allSinks.remove(router.categorySinks.get(exclude));
+		allSinks.removeAll(router.categorySinks.get(exclude));
 		for(AuditLogSink sink : allSinks) {
 			LoggingSink loggingSink = (LoggingSink)sink;
 			Assert.assertEquals(0, loggingSink.messages.size());

src/test/java/com/amazon/opendistroforelasticsearch/security/auditlog/sink/KafkaSinkTest.java

@@ -40,7 +40,7 @@
 public class KafkaSinkTest extends AbstractAuditlogiUnitTest {
 
 	@ClassRule
-	public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, 1, "compliance");
+	public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 1, "compliance");
 
 	@Test
 	public void testKafka() throws Exception {
@@ -57,7 +57,7 @@ public void testKafka() throws Exception {
 	            Assert.assertEquals(KafkaSink.class, sink.getClass());
     	        boolean success = sink.doStore(MockAuditMessageFactory.validAuditMessage(Category.MISSING_PRIVILEGES));
     	        Assert.assertTrue(success);
-    	        ConsumerRecords<Long, String> records = consumer.poll(10000);
+    	        ConsumerRecords<Long, String> records = consumer.poll(Duration.ofSeconds(10));
     	        Assert.assertEquals(1, records.count());
 	        } finally {
 	            sink.close();
@@ -68,7 +68,7 @@ public void testKafka() throws Exception {
 
 	private KafkaConsumer<Long, String> createConsumer() {
 		Properties props = new Properties();
-		props.put("bootstrap.servers", embeddedKafka.getBrokersAsString());
+		props.put("bootstrap.servers", embeddedKafka.getEmbeddedKafka().getBrokersAsString());
 		props.put("auto.offset.reset", "earliest");
 		props.put("group.id", "mygroup"+System.currentTimeMillis()+"_"+new Random().nextDouble());
 		props.put("key.deserializer", "org.apache.kafka.common.serialization.LongDeserializer");

src/test/java/com/amazon/opendistroforelasticsearch/security/dlic/dlsfls/DlsScrollTest.java

@@ -0,0 +1,67 @@
+package com.amazon.opendistroforelasticsearch.security.dlic.dlsfls;
+
+import org.apache.http.HttpStatus;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.floragunn.searchguard.test.helper.file.FileHelper;
+import com.floragunn.searchguard.test.helper.rest.RestHelper.HttpResponse;
+
+public class DlsScrollTest extends AbstractDlsFlsTest{
+
+
+    @Override
+    protected void populateData(TransportClient tc) {
+
+        tc.index(new IndexRequest("deals").type("deals").id("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .source("{\"amount\": 3}", XContentType.JSON)).actionGet(); //not in
+
+        tc.index(new IndexRequest("deals").type("deals").id("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .source("{\"amount\": 10}", XContentType.JSON)).actionGet(); //not in
+
+        tc.index(new IndexRequest("deals").type("deals").id("2").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .source("{\"amount\": 1500}", XContentType.JSON)).actionGet();
+
+        tc.index(new IndexRequest("deals").type("deals").id("4").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .source("{\"amount\": 21500}", XContentType.JSON)).actionGet(); //not in
+
+        for(int i=0; i<100; i++) {
+            tc.index(new IndexRequest("deals").type("deals").id("gen"+i).setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                    .source("{\"amount\": 1500}", XContentType.JSON)).actionGet();
+        }
+    }
+
+
+    @Test
+    public void testDlsScroll() throws Exception {
+
+        setup();
+
+        HttpResponse res;
+        Assert.assertEquals(HttpStatus.SC_OK, (res=rh.executeGetRequest("/deals/_search?scroll=1m&pretty=true&size=5", encodeBasicHeader("dept_manager", "password"))).getStatusCode());
+        Assert.assertTrue(res.getBody().contains("\"value\" : 101,"));
+
+        int c=0;
+
+        while(true) {
+            int start = res.getBody().indexOf("_scroll_id") + 15;
+            String scrollid = res.getBody().substring(start, res.getBody().indexOf("\"", start+1));
+            Assert.assertEquals(HttpStatus.SC_OK, (res=rh.executePostRequest("/_search/scroll?pretty=true", "{\"scroll\" : \"1m\", \"scroll_id\" : \""+scrollid+"\"}", encodeBasicHeader("dept_manager", "password"))).getStatusCode());
+            Assert.assertTrue(res.getBody().contains("\"value\" : 101,"));
+            Assert.assertFalse(res.getBody().contains("\"amount\" : 3"));
+            Assert.assertFalse(res.getBody().contains("\"amount\" : 10"));
+            Assert.assertFalse(res.getBody().contains("\"amount\" : 21500"));
+            c++;
+
+            if(res.getBody().contains("\"hits\" : [ ]")) {
+                break;
+            }
+        }
+
+        Assert.assertEquals(21, c);
+    }
+}