From cd0040e913677148034aac52716a733be77f6d5e Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 30 Apr 2024 00:41:50 +0000 Subject: [PATCH] make security-analytics integTest stable Signed-off-by: Subhobrata Dey --- .../CorrelationEngineRestApiIT.java | 557 +++++++++--------- .../integTests/ThreatIntelJobRunnerIT.java | 2 +- 2 files changed, 266 insertions(+), 293 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index a4cdb6d1c..a2979a231 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -23,12 +23,15 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.test.rest.OpenSearchRestTestCase; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; import java.util.stream.Collectors; import static org.opensearch.securityanalytics.TestHelpers.*; @@ -88,33 +91,30 @@ public void testBasicCorrelationEngineWorkflow() throws IOException, Interrupted Map getFindingsBody = entityAsMap(getFindingsResponse); String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 2) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 2) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + return true; } + return false; + } catch (Exception ex) { + return true; } - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + }, + 2, TimeUnit.MINUTES + ); } @SuppressWarnings("unchecked") @@ -143,29 +143,26 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept Thread.sleep(5000); - int count = 0; - while (true) { - try { - Long endTime = System.currentTimeMillis(); - Request request = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); - Response response = client().performRequest(request); - - Map responseMap = entityAsMap(response); - List results = (List) responseMap.get("findings"); - if (results.size() == 1) { - Assert.assertTrue(true); - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + OpenSearchRestTestCase.waitUntil( + () -> { + try { + Long endTime = System.currentTimeMillis(); + Request request = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); + Response response = client().performRequest(request); + + Map responseMap = entityAsMap(response); + List results = (List) responseMap.get("findings"); + if (results.size() == 1) { + Assert.assertTrue(true); + return true; + } + return false; + } catch (Exception ex) { + return false; + } + }, + 2, TimeUnit.MINUTES + ); } @SuppressWarnings("unchecked") @@ -216,24 +213,21 @@ public void testBasicCorrelationEngineWorkflowWithoutRules() throws IOException, Map getFindingsBody = entityAsMap(getFindingsResponse); String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 2) { - Assert.assertTrue(true); - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + OpenSearchRestTestCase.waitUntil( + () -> { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 2) { + Assert.assertTrue(true); + return true; + } + return false; + } catch (Exception ex) { + return false; + } + }, + 2, TimeUnit.MINUTES + ); } @SuppressWarnings("unchecked") @@ -296,40 +290,40 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxAge() throws IOEx String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); Thread.sleep(1000L); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 2) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + OpenSearchRestTestCase.waitUntil( + new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 2) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + + List correlationIndices = getCorrelationHistoryIndices(); + while (correlationIndices.size() < 2) { + correlationIndices = getCorrelationHistoryIndices(); + Thread.sleep(1000); + } + Assert.assertTrue("Did not find more then 2 correlation indices", correlationIndices.size() >= 2); + return true; + } + return false; + } catch (Exception ex) { + return false; } } - - List correlationIndices = getCorrelationHistoryIndices(); - while (correlationIndices.size() < 2) { - correlationIndices = getCorrelationHistoryIndices(); - Thread.sleep(1000); - } - Assert.assertTrue("Did not find more then 2 correlation indices", correlationIndices.size() >= 2); - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + }, + 2, TimeUnit.MINUTES + ); } public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDoc() throws IOException, InterruptedException { @@ -391,40 +385,37 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDoc() throws IOEx String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); Thread.sleep(1000L); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 2) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 2) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + + List correlationIndices = getCorrelationHistoryIndices(); + while (correlationIndices.size() < 2) { + correlationIndices = getCorrelationHistoryIndices(); + Thread.sleep(1000); + } + Assert.assertTrue("Did not find more then 2 correlation indices", correlationIndices.size() >= 2); + return true; } + return false; + } catch (Exception ex) { + return false; } - - List correlationIndices = getCorrelationHistoryIndices(); - while (correlationIndices.size() < 2) { - correlationIndices = getCorrelationHistoryIndices(); - Thread.sleep(1000); - } - Assert.assertTrue("Did not find more then 2 correlation indices", correlationIndices.size() >= 2); - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + }, + 2, TimeUnit.MINUTES + ); } public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDocAndShortRetention() throws IOException, InterruptedException { @@ -486,49 +477,46 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDocAndShortRetent String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); Thread.sleep(1000L); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 2) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 2) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + + List correlationIndices = getCorrelationHistoryIndices(); + while (correlationIndices.size() < 2) { + correlationIndices = getCorrelationHistoryIndices(); + Thread.sleep(1000); + } + Assert.assertTrue("Did not find more then 2 correlation indices", correlationIndices.size() >= 2); + + updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD.getKey(), "1s"); + updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_MAX_DOCS.getKey(), "1000"); + + while (correlationIndices.size() != 1) { + correlationIndices = getCorrelationHistoryIndices(); + Thread.sleep(1000); + } + Assert.assertTrue("Found more than 1 correlation indices", correlationIndices.size() == 1); + return true; } + return false; + } catch (Exception ex) { + return false; } - - List correlationIndices = getCorrelationHistoryIndices(); - while (correlationIndices.size() < 2) { - correlationIndices = getCorrelationHistoryIndices(); - Thread.sleep(1000); - } - Assert.assertTrue("Did not find more then 2 correlation indices", correlationIndices.size() >= 2); - - updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD.getKey(), "1s"); - updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_MAX_DOCS.getKey(), "1000"); - - while (correlationIndices.size() != 1) { - correlationIndices = getCorrelationHistoryIndices(); - Thread.sleep(1000); - } - Assert.assertTrue("Found more than 1 correlation indices", correlationIndices.size() == 1); - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + }, + 2, TimeUnit.MINUTES + ); } public void testBasicCorrelationEngineWorkflowWithFieldBasedRules() throws IOException, InterruptedException { @@ -614,29 +602,26 @@ public void testBasicCorrelationEngineWorkflowWithFieldBasedRules() throws IOExc Thread.sleep(5000); - int count = 0; - while (true) { - try { - Long endTime = System.currentTimeMillis(); - Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); - response = client().performRequest(restRequest); - - Map responseMap = entityAsMap(response); - List results = (List) responseMap.get("findings"); - if (results.size() == 1) { - Assert.assertTrue(true); - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + OpenSearchRestTestCase.waitUntil( + () -> { + try { + Long endTime = System.currentTimeMillis(); + Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); + Response restResponse = client().performRequest(restRequest); + + Map responseMap = entityAsMap(restResponse); + List results = (List) responseMap.get("findings"); + if (results.size() == 1) { + Assert.assertTrue(true); + return true; + } + return false; + } catch (Exception ex) { + return false; + } + }, + 2, TimeUnit.MINUTES + ); } public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesOnMultipleLogTypes() throws IOException, InterruptedException { @@ -671,33 +656,30 @@ public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesOnMultipleLogTy Map getFindingsBody = entityAsMap(getFindingsResponse); String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 1) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 1) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + return true; + } + } } + return false; + } catch (Exception ex) { + return false; } - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + }, + 2, TimeUnit.MINUTES + ); } public void testBasicCorrelationEngineWorkflowWithIndexPatterns() throws IOException, InterruptedException { @@ -736,33 +718,30 @@ public void testBasicCorrelationEngineWorkflowWithIndexPatterns() throws IOExce Map getFindingsBody = entityAsMap(getFindingsResponse); String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 1) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 1) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + return true; + } + } } + return false; + } catch (Exception ex) { + return false; } - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + }, + 2, TimeUnit.MINUTES + ); } public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesAndDynamicTimeWindow() throws IOException, InterruptedException { @@ -848,29 +827,26 @@ public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesAndDynamicTimeW Thread.sleep(5000); - int count = 0; - while (true) { - try { - Long endTime = System.currentTimeMillis(); - Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); - response = client().performRequest(restRequest); - - Map responseMap = entityAsMap(response); - List results = (List) responseMap.get("findings"); - if (results.size() == 1) { - Assert.assertTrue(true); - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 2) { - break; - } - } - Assert.assertEquals(2, count); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + Long endTime = System.currentTimeMillis(); + Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); + Response response1 = client().performRequest(restRequest); + + Map responseMap = entityAsMap(response1); + List results = (List) responseMap.get("findings"); + if (results.size() == 1) { + Assert.assertTrue(true); + return true; + } + return false; + } catch (Exception ex) { + return false; + } + }, + 2, TimeUnit.MINUTES + ); } public void testBasicCorrelationEngineWorkflowWithCustomLogTypes() throws IOException, InterruptedException { @@ -952,33 +928,30 @@ public void testBasicCorrelationEngineWorkflowWithCustomLogTypes() throws IOExce Map getFindingsBody = entityAsMap(getFindingsResponse); String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); - int count = 0; - while (true) { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, customLogType.getName(), 300000L, 10); - if (correlatedFindings.size() == 1) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + OpenSearchRestTestCase.waitUntil( + () -> { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, customLogType.getName(), 300000L, 10); + if (correlatedFindings.size() == 1) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + return true; + } + } } + return false; + } catch (Exception ex) { + return false; } - break; - } - } catch (Exception ex) { - // suppress ex - } - ++count; - Thread.sleep(5000); - if (count >= 12) { - Assert.assertTrue(false); - break; - } - } + }, + 2, TimeUnit.MINUTES + ); } private LogIndices createIndices() throws IOException { diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java index cf4cc800c..2a268a264 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java @@ -141,7 +141,7 @@ public void testCreateDetector_threatIntelEnabled_testJobRunner() throws IOExcep } catch (IOException e) { throw new RuntimeException("failed to verify that job ran"); } - }, 120, TimeUnit.SECONDS); + }, 240, TimeUnit.SECONDS); // verify job's last update time is different List newJobMetaDataList = getJobSchedulerParameter();