From decee51de3206b4baf6ecdfa86b872607ab310b2 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Mon, 16 Oct 2023 16:12:47 -0700 Subject: [PATCH] fix compilation issues in tests Signed-off-by: Surya Sashank Nistala --- .../LogTypeServiceTests.java | 3 +- .../model/WriteableTests.java | 8 +- .../resthandler/DetectorMonitorRestApiIT.java | 161 +++--------------- .../writable/LogTypeTests.java | 13 +- 4 files changed, 44 insertions(+), 141 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java index 8eb717e60..64288f669 100644 --- a/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java @@ -50,7 +50,8 @@ protected void beforeTest() throws Exception { new LogType.Mapping("rawFld1", "ecsFld1", "ocsfFld1"), new LogType.Mapping("rawFld2", "ecsFld2", "ocsfFld2"), new LogType.Mapping("rawFld3", "ecsFld3", "ocsfFld3") - ) + ), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ) ); when(builtinLogTypeLoader.getAllLogTypes()).thenReturn(dummyLogTypes); diff --git a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java index e82911c1b..7c16e5f6f 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java @@ -50,7 +50,8 @@ public void testEmptyUserAsStream() throws IOException { public void testLogTypeAsStreamRawFieldOnly() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", null, null)) + List.of(new LogType.Mapping("rawField", null, null)), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -66,7 +67,8 @@ public void testLogTypeAsStreamRawFieldOnly() throws IOException { public void testLogTypeAsStreamFull() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")) + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -80,7 +82,7 @@ public void testLogTypeAsStreamFull() throws IOException { } public void testLogTypeAsStreamNoMappings() throws IOException { - LogType logType = new LogType("1", "my_log_type", "description", false, null); + LogType logType = new LogType("1", "my_log_type", "description", false, null, null); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index f46fd0efb..a4a38274f 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1052,7 +1052,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule verifyWorkflow(detectorMap, monitorIds, 2); } - public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatIntel() throws IOException { + public void testCreateDetectorWiththreatIntelEnabled_updateDetectorWithThreatIntelDisabled() throws IOException { updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true"); String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -1136,143 +1136,20 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt String threatIntelDocLevelQueryId = docLevelQueryResults.keySet().stream().filter(id -> id.contains(detector.getName() + "_threat_intel")).findAny().get(); ArrayList docs = (ArrayList) docLevelQueryResults.get(threatIntelDocLevelQueryId); assertEquals(docs.size(), 3); -// -// Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(detector)); -// -// assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); -// -// Map updateResponseBody = asMap(updateResponse); -// detectorId = updateResponseBody.get("_id").toString(); -// -// indexDoc(index, "4", randomDoc(5, 3, "klm")); -// -// executeResponse = executeAlertingWorkflow(workflowId, Collections.emptyMap()); -// -// monitorRunResults = (List>) entityAsMap(executeResponse).get("monitor_run_results"); -// assertEquals(1, monitorRunResults.size()); -// -// docLevelQueryResults = ((List>) ((Map) monitorRunResults.get(0).get("input_results")).get("results")).get(0); -// noOfSigmaRuleMatches = docLevelQueryResults.size(); -// assertEquals(2, noOfSigmaRuleMatches); -// threatIntelDocLevelQueryId = docLevelQueryResults.keySet().stream().filter(id -> id.contains(detector.getName() + "_threat_intel")).findAny().get(); -// docs = (ArrayList) docLevelQueryResults.get(threatIntelDocLevelQueryId); -// assertEquals(docs.size(), 1); - } - - private List getThreatIntelFeedIocs(int num) throws IOException { - String request = getMatchAllSearchRequestString(num); - SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel*", request, false); - return getTifdList(res, xContentRegistry()).stream().map(it -> it.getIocValue()).collect(Collectors.toList()); - } - - private static String getMatchAllSearchRequestString(int num) { - return "{\n" + - "\"size\" : " + num + "," + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - } - - - public void testCreateDetectorthreatIntelDisabled_updateDetectorWithThreatIntelEnabled() throws IOException { - String tifdString1 = "{ \"type\": \"feed\",\"ioc_type\": \"ip\", \"ioc_value\": \"abc\", \"feed_id\": \"feed\", \"timestamp\": 1633344000000 }"; - String tifdString2 = "{ \"type\": \"feed\",\"ioc_type\": \"ip\", \"ioc_value\": \"xyz\", \"feed_id\": \"feed\", \"timestamp\": 1633344000000 }"; - String feedIndex = ".opensearch-sap-threatintel"; - indexDoc(feedIndex, "1", tifdString1); - indexDoc(feedIndex, "2", tifdString2); - updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true"); - String index = createTestIndex(randomIndex(), windowsIndexMapping()); - // Execute CreateMappingsAction to add alias mapping for index - Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - // both req params and req body are supported - createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" - ); - - Response createMappingResponse = client().performRequest(createMappingRequest); - - assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); - - String testOpCode = "Test"; - - String randomDocRuleId = createRule(randomRule()); - List detectorRules = List.of(new DetectorRule(randomDocRuleId)); - DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, - Collections.emptyList()); - Detector detector = randomDetectorWithInputsAndThreatIntel(List.of(input), false); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(1, response.getHits().getTotalHits().value); - - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - Map responseBody = asMap(createResponse); - - String detectorId = responseBody.get("_id").toString(); - request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); - List inputArr = (List) detectorMap.get("inputs"); - - - List monitorIds = ((List) (detectorMap).get("monitor_id")); - assertEquals(1, monitorIds.size()); - - assertNotNull("Workflow not created", detectorMap.get("workflow_ids")); - assertEquals("Number of workflows not correct", 1, ((List) detectorMap.get("workflow_ids")).size()); - - // Verify workflow - verifyWorkflow(detectorMap, monitorIds, 1); - - indexDoc(index, "1", randomDoc(5, 3, "abc")); - indexDoc(index, "2", randomDoc(5, 3, "xyz")); - indexDoc(index, "3", randomDoc(5, 3, "klm")); - String workflowId = ((List) detectorMap.get("workflow_ids")).get(0); - - Response executeResponse = executeAlertingWorkflow(workflowId, Collections.emptyMap()); - - List> monitorRunResults = (List>) entityAsMap(executeResponse).get("monitor_run_results"); - assertEquals(1, monitorRunResults.size()); - - Map docLevelQueryResults = ((List>) ((Map) monitorRunResults.get(0).get("input_results")).get("results")).get(0); - int noOfSigmaRuleMatches = docLevelQueryResults.size(); - assertEquals(1, noOfSigmaRuleMatches); - - - //update threat intel - String tifdString3 = "{ \"type\": \"feed\",\"ioc_type\": \"ip\", \"ioc_value\": \"klm\", \"feed_id\": \"feed\", \"timestamp\": 1633344000000 }"; - - indexDoc(feedIndex, "3", tifdString3); - detector.setThreatIntelEnabled(true); + detector.setThreatIntelEnabled(false); Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(detector)); assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); Map updateResponseBody = asMap(updateResponse); - detectorId = updateResponseBody.get("_id").toString(); - - indexDoc(index, "4", randomDoc(5, 3, "klm")); + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + assertEquals(1, response.getHits().getTotalHits().value); //threat intel based queries should not be present as threat intel is disabled. + i=1; + for (String ioc : iocs) { + indexDoc(index, i+"", randomDocWithIpIoc(5, 3, ioc)); + i++; + } executeResponse = executeAlertingWorkflow(workflowId, Collections.emptyMap()); @@ -1282,11 +1159,27 @@ public void testCreateDetectorthreatIntelDisabled_updateDetectorWithThreatIntelE docLevelQueryResults = ((List>) ((Map) monitorRunResults.get(0).get("input_results")).get("results")).get(0); noOfSigmaRuleMatches = docLevelQueryResults.size(); assertEquals(2, noOfSigmaRuleMatches); - String threatIntelDocLevelQueryId = docLevelQueryResults.keySet().stream().filter(id -> id.contains(detector.getName() + "_threat_intel")).findAny().get(); - ArrayList docs = (ArrayList) docLevelQueryResults.get(threatIntelDocLevelQueryId); + threatIntelDocLevelQueryId = docLevelQueryResults.keySet().stream().filter(id -> id.contains(detector.getName() + "_threat_intel")).findAny().get(); + docs = (ArrayList) docLevelQueryResults.get(threatIntelDocLevelQueryId); assertEquals(docs.size(), 1); } + private List getThreatIntelFeedIocs(int num) throws IOException { + String request = getMatchAllSearchRequestString(num); + SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel*", request, false); + return getTifdList(res, xContentRegistry()).stream().map(it -> it.getIocValue()).collect(Collectors.toList()); + } + + private static String getMatchAllSearchRequestString(int num) { + return "{\n" + + "\"size\" : " + num + "," + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + } + public void testCreateDetector_verifyWorkflowCreation_success_WithGroupByRulesInTrigger() throws IOException { updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true"); String index = createTestIndex(randomIndex(), windowsIndexMapping()); diff --git a/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java index 4ede7891b..d9d592641 100644 --- a/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java +++ b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java @@ -21,7 +21,8 @@ public class LogTypeTests { public void testLogTypeAsStreamRawFieldOnly() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", null, null)) + List.of(new LogType.Mapping("rawField", null, null)), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -32,13 +33,16 @@ public void testLogTypeAsStreamRawFieldOnly() throws IOException { assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + assertEquals(logType.getIocFieldsList().get(0).getFields().get(0), newLogType.getIocFieldsList().get(0).getFields().get(0)); + assertEquals(logType.getIocFieldsList().get(0).getIoc(), newLogType.getIocFieldsList().get(0).getIoc()); } @Test public void testLogTypeAsStreamFull() throws IOException { LogType logType = new LogType( "1", "my_log_type", "description", false, - List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")) + List.of(new LogType.Mapping("rawField", "some_ecs_field", "some_ocsf_field")), + List.of(new LogType.IocFields("ip", List.of("dst.ip"))) ); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); @@ -49,11 +53,14 @@ public void testLogTypeAsStreamFull() throws IOException { assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); assertEquals(logType.getMappings().get(0).getRawField(), newLogType.getMappings().get(0).getRawField()); + assertEquals(logType.getIocFieldsList().get(0).getFields().get(0), newLogType.getIocFieldsList().get(0).getFields().get(0)); + assertEquals(logType.getIocFieldsList().get(0).getIoc(), newLogType.getIocFieldsList().get(0).getIoc()); + } @Test public void testLogTypeAsStreamNoMappings() throws IOException { - LogType logType = new LogType("1", "my_log_type", "description", false, null); + LogType logType = new LogType("1", "my_log_type", "description", false, null, null); BytesStreamOutput out = new BytesStreamOutput(); logType.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes);