From 29cb35b0bf6c0966abcfe9cbf3ab8d0135bb2aa2 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 4 Jun 2024 02:13:33 +0000 Subject: [PATCH 1/5] fix upgrade issues due to lucene codec name change (#1047) Signed-off-by: Subhobrata Dey --- .../correlation/index/codec/CorrelationCodecVersion.java | 2 +- .../transport/TransportIndexDetectorAction.java | 8 ++++---- .../opensearch/securityanalytics/util/DetectorUtils.java | 2 +- .../securityanalytics/alerts/AlertingServiceTests.java | 4 ++-- .../org/opensearch/securityanalytics/alerts/AlertsIT.java | 1 + 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java index 07721ae9b..c6ffd8551 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java @@ -20,7 +20,7 @@ public enum CorrelationCodecVersion { V_9_5_0( - "CorrelationCodec950", + "CorrelationCodec", new Lucene95Codec(), new PerFieldCorrelationVectorsFormat950(Optional.empty()), (userCodec, mapperService) -> new CorrelationCodec950(userCodec, new PerFieldCorrelationVectorsFormat950(Optional.of(mapperService))), diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index ebe7d022d..7995c14b6 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -785,7 +785,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -886,7 +886,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -1060,7 +1060,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { } **/ Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), + MonitorType.BUCKET_LEVEL_MONITOR.getValue(), detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -1782,7 +1782,7 @@ private Map mapMonitorIds(List monitorResp Collectors.toMap( // In the case of bucket level monitors rule id is trigger id it -> { - if (MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType()) { + if (MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(it.getMonitor().getMonitorType())) { return it.getMonitor().getTriggers().get(0).getId(); } else { if (it.getMonitor().getName().contains("_chained_findings")) { diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java index 14c241f83..2a3bb041a 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java @@ -109,7 +109,7 @@ public static List getBucketLevelMonitorIds( ) { return monitorResponses.stream().filter( // In the case of bucket level monitors rule id is trigger id - it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType() + it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(it.getMonitor().getMonitorType()) ).map(IndexMonitorResponse::getId).collect(Collectors.toList()); } public static List getAggRuleIdsConfiguredToTrigger(Detector detector, List> rulesById) { diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 1c928d2fc..06e464d34 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -88,7 +88,7 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), @@ -122,7 +122,7 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index e6f4eff6d..f9f3e25d5 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -179,6 +179,7 @@ public void testGetAlerts_success() throws IOException { assertEquals(((ArrayList) ackAlertsResponseMap.get("acknowledged")).size(), 1); } + @Ignore @SuppressWarnings("unchecked") public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); From 1547393eb56828431fc0214e416d616722b9b5c1 Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Mon, 10 Jun 2024 14:10:46 -0700 Subject: [PATCH 2/5] address the design changes discussed Signed-off-by: Riya Saxena --- .../securityanalytics/correlation/JoinEngine.java | 1 - .../correlation/alert/CorrelationRuleScheduler.java | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java index 03d4a0b73..20cff273a 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java @@ -560,7 +560,6 @@ private void getCorrelatedFindings(String detectorType, Map if (!correlatedFindings.isEmpty()) { CorrelationRuleScheduler correlationRuleScheduler = new CorrelationRuleScheduler(client, correlationAlertService, notificationService); correlationRuleScheduler.schedule(correlationRules, correlatedFindings, request.getFinding().getId(), indexTimeout, user); - correlationRuleScheduler.shutdown(); } for (Map.Entry> autoCorrelation: autoCorrelations.entrySet()) { diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationRuleScheduler.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationRuleScheduler.java index 945407b15..ba42e252b 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationRuleScheduler.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationRuleScheduler.java @@ -22,8 +22,6 @@ import java.util.List; import java.util.ArrayList; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; public class CorrelationRuleScheduler { @@ -31,13 +29,11 @@ public class CorrelationRuleScheduler { private final Client client; private final CorrelationAlertService correlationAlertService; private final NotificationService notificationService; - private final ExecutorService executorService; public CorrelationRuleScheduler(Client client, CorrelationAlertService correlationAlertService, NotificationService notificationService) { this.client = client; this.correlationAlertService = correlationAlertService; this.notificationService = notificationService; - this.executorService = Executors.newCachedThreadPool(); } public void schedule(List correlationRules, Map> correlatedFindings, String sourceFinding, TimeValue indexTimeout, User user) { @@ -56,15 +52,11 @@ public void schedule(List correlationRules, Map findingIds, TimeValue indexTimeout, String sourceFindingId, User user) { long startTime = Instant.now().toEpochMilli(); long endTime = startTime + correlationRule.getCorrTimeWindow(); RuleTask ruleTask = new RuleTask(correlationRule, findingIds, startTime, endTime, correlationAlertService, notificationService, indexTimeout, sourceFindingId, user); - executorService.submit(ruleTask); + ruleTask.run(); } private class RuleTask implements Runnable { From abade3fb8d61184e697418450aaa3f4398f2ff4d Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Mon, 10 Jun 2024 14:22:16 -0700 Subject: [PATCH 3/5] address the design changes discussed Signed-off-by: Riya Saxena --- .../SecurityAnalyticsPlugin.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index f18f75639..f97afcb60 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -52,7 +52,30 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; -import org.opensearch.securityanalytics.action.*; +import org.opensearch.securityanalytics.action.GetAlertsAction; +import org.opensearch.securityanalytics.action.DeleteCorrelationRuleAction; +import org.opensearch.securityanalytics.action.AckAlertsAction; +import org.opensearch.securityanalytics.action.CreateIndexMappingsAction; +import org.opensearch.securityanalytics.action.CorrelatedFindingAction; +import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.action.DeleteDetectorAction; +import org.opensearch.securityanalytics.action.DeleteRuleAction; +import org.opensearch.securityanalytics.action.GetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.action.GetDetectorAction; +import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.action.GetIndexMappingsAction; +import org.opensearch.securityanalytics.action.GetMappingsViewAction; +import org.opensearch.securityanalytics.action.IndexCorrelationRuleAction; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeAction; +import org.opensearch.securityanalytics.action.IndexDetectorAction; +import org.opensearch.securityanalytics.action.IndexRuleAction; +import org.opensearch.securityanalytics.action.ListCorrelationsAction; +import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction; +import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; +import org.opensearch.securityanalytics.action.SearchDetectorAction; +import org.opensearch.securityanalytics.action.SearchRuleAction; +import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction; +import org.opensearch.securityanalytics.action.ValidateRulesAction; import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService; import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; From c3476fe24d0dcd64e3dc6e95d4e308d68fd1c408 Mon Sep 17 00:00:00 2001 From: Chase <62891993+engechas@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:58:05 -0700 Subject: [PATCH 4/5] Add 2.15 release notes (#1061) Signed-off-by: Chase Engelbrecht --- ...h-security-analytics.release-notes-2.15.0.0.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md diff --git a/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md new file mode 100644 index 000000000..a93be647e --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md @@ -0,0 +1,15 @@ +## Version 2.15.0.0 2024-06-10 + +Compatible with OpenSearch 2.15.0 + +### Maintenance +* Increment version to 2.15.0-SNAPSHOT. ([#1055](https://github.com/opensearch-project/security-analytics/pull/1055)) +* Fix codecov calculation ([#1021](https://github.com/opensearch-project/security-analytics/pull/1021)) +* Stabilize integ tests ([#1014](https://github.com/opensearch-project/security-analytics/pull/1014)) + +### Bug Fixes +* Fix chained findings monitor logic in update detector flow ([#1019](https://github.com/opensearch-project/security-analytics/pull/1019)) +* Change default filter to time based fields ([#1030](https://github.com/opensearch-project/security-analytics/pull/1030)) + +### Documentation +* Added 2.15.0 release notes. ([#1061](https://github.com/opensearch-project/security-analytics/pull/1061)) \ No newline at end of file From 5ca18991cb25b1646746a3bc3ff89ad529eb19f4 Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Tue, 11 Jun 2024 13:50:06 -0700 Subject: [PATCH 5/5] fixed tests Signed-off-by: Riya Saxena --- .../opensearch/securityanalytics/TestHelpers.java | 2 +- .../correlation/CorrelationEngineRestApiIT.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index a1987138d..33d0de4cc 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -227,7 +227,7 @@ public static CorrelationRule randomCorrelationRule(String name) { List.of( new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null), new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap", null) - ), 300000L); + ), 300000L, null); } public static String randomRule() { diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index a2979a231..a7eda56aa 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -968,7 +968,7 @@ private String createNetworkToWindowsFieldBasedRule(LogIndices indices) throws I CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp"); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -981,7 +981,7 @@ private String createNetworkToWindowsFilterQueryBasedRule(LogIndices indices) th CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "srcaddr:1.2.3.4", "network", null); CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "SourceIp:1.2.3.4", "test_windows", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -994,7 +994,7 @@ private String createNetworkToCustomLogTypeFieldBasedRule(LogIndices indices, St CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); CorrelationQuery query4 = new CorrelationQuery(customLogTypeIndex, null, customLogTypeName, "SourceIp"); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -1008,7 +1008,7 @@ private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOE CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null); CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -1022,7 +1022,7 @@ private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOE CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null); CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -1035,7 +1035,7 @@ private String createCloudtrailFieldBasedRule(String index, String field, Long t CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field); CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow, null); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request);