Skip to content

Commit

Permalink
separate doc-level monitor query indices created by detectors (#1324)
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <[email protected]>
(cherry picked from commit 1ca4090)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@github-actions
github-actions[bot] committed Sep 26, 2024
1 parent 2958d18 commit 18e3ef2
Show file tree
Hide file tree
Showing 19 changed files with 343 additions and 145 deletions.
3 changes: 2 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,8 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD,
SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN,
SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT
SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT,
SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES
);
}

Expand Down
6 changes: 6 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
package org.opensearch.securityanalytics.config.monitors;

import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
import org.opensearch.common.inject.Inject;
import org.opensearch.securityanalytics.logtype.LogTypeService;
Expand All @@ -25,6 +27,10 @@ public static String getRuleIndex(String logType) {
return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType);
}

public static String getRuleIndexOptimized(String logType) {
return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID());
}

public static String getAlertsIndex(String logType) {
return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType);
}
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,4 +249,10 @@ public static final List<Setting<?>> settings() {
Setting.Property.NodeScope, Setting.Property.Dynamic
);

public static final Setting<Boolean> ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES = Setting.boolSetting(
"plugins.security_analytics.enable_detectors_with_dedicated_query_indices",
false,
Setting.Property.NodeScope, Setting.Property.Dynamic
);

}
1 change: 1 addition & 0 deletions ...curityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request)
triggers,
Collections.emptyMap(),
new DataSources(),
false,
PLUGIN_OWNER_FIELD
);
} catch (Exception e) {
Expand Down
35 changes: 29 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction<IndexDe

private volatile Boolean enabledWorkflowUsage;

private volatile Boolean enableDetectorWithDedicatedQueryIndices;

private final Settings settings;

private final NamedWriteableRegistry namedWriteableRegistry;
Expand Down Expand Up @@ -201,11 +203,13 @@ public TransportIndexDetectorAction(TransportService transportService,
this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings);
this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings);
this.enabledWorkflowUsage = SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE.get(this.settings);
this.enableDetectorWithDedicatedQueryIndices = SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES.get(this.settings);
this.monitorService = new MonitorService(client);
this.workflowService = new WorkflowService(client, monitorService);

this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled);
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, this::setEnabledWorkflowUsage);
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES, this::setEnabledDetectorsWithDedicatedQueryIndices);
this.exceptionChecker = exceptionChecker;
}

Expand Down Expand Up @@ -792,7 +796,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List<Pair<String, Rule>
detector.getAlertsHistoryIndex(),
detector.getAlertsHistoryIndexPattern(),
DetectorMonitorConfig.getRuleIndexMappingsByType(),
true), PLUGIN_OWNER_FIELD);
true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD);

return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null);
}
Expand Down Expand Up @@ -886,14 +890,14 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest(

Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null,
Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(),
new DataSources(detector.getRuleIndex(),
new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(),
detector.getFindingsIndex(),
detector.getFindingsIndexPattern(),
detector.getAlertsIndex(),
detector.getAlertsHistoryIndex(),
detector.getAlertsHistoryIndexPattern(),
DetectorMonitorConfig.getRuleIndexMappingsByType(),
true), PLUGIN_OWNER_FIELD);
true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD);

return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null);
}
Expand Down Expand Up @@ -1063,7 +1067,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) {
detector.getAlertsHistoryIndex(),
detector.getAlertsHistoryIndexPattern(),
DetectorMonitorConfig.getRuleIndexMappingsByType(),
true), PLUGIN_OWNER_FIELD);
true), false, PLUGIN_OWNER_FIELD);

listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null));
}
Expand Down Expand Up @@ -1247,7 +1251,13 @@ void createDetector() {
request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic));
request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic));
request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic));
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));

if (enableDetectorWithDedicatedQueryIndices) {
// disabling the setting after enabling it will mean delete & re-create the detector
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic));
} else {
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));
}

User originalContextUser = this.user;
log.debug("user from original context is {}", originalContextUser);
Expand Down Expand Up @@ -1364,7 +1374,16 @@ void onGetResponse(Detector currentDetector, User user) {
request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic));
request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic));
request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic));
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));
if (currentDetector.getRuleIndex().contains("optimized")) {
request.getDetector().setRuleIndex(currentDetector.getRuleIndex());
} else {
if (enableDetectorWithDedicatedQueryIndices) {
// disabling the setting after enabling it will mean delete & re-create the detector
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic));
} else {
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic));

Check warning on line 1384 in src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java#L1384 

Added line #L1384 was not covered by tests
}
}
request.getDetector().setUser(user);

if (!detector.getInputs().isEmpty()) {
Expand Down Expand Up @@ -1800,4 +1819,8 @@ private void setFilterByEnabled(boolean filterByEnabled) {
private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) {
this.enabledWorkflowUsage = enabledWorkflowUsage;
}

private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) {
this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices;
}
}
9 changes: 3 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,9 @@ public static String ruleTopicIndexSettings() throws IOException {
public void initRuleTopicIndexTemplate(ActionListener<AcknowledgedResponse> actionListener) throws IOException {
getAllRuleIndices(ActionListener.wrap(allRuleIndices -> {
// Compose list of all patterns to cover all query indices
List<String> indexPatterns = new ArrayList<>();
for(String ruleIndex : allRuleIndices) {
indexPatterns.add(ruleIndex + "*");
}

ComposableIndexTemplate template = new ComposableIndexTemplate(
indexPatterns,
allRuleIndices,
new Template(
Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(),
null,
Expand Down Expand Up @@ -87,7 +83,8 @@ private void getAllRuleIndices(ActionListener<List<String>> listener) {
listener.onResponse(
logTypes
.stream()
.map(logType -> DetectorMonitorConfig.getRuleIndex(logType))
// use index pattern here to define rule topic index template for all query indices which match the pattern
.map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*")
.collect(Collectors.toList())
);
}, listener::onFailure));
Expand Down
16 changes: 8 additions & 8 deletions src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);


assertEquals(2, response.getHits().getTotalHits().value);
Expand Down Expand Up @@ -275,7 +275,7 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);


assertEquals(1, response.getHits().getTotalHits().value);
Expand Down Expand Up @@ -372,7 +372,7 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);


assertEquals(1, response.getHits().getTotalHits().value);
Expand Down Expand Up @@ -466,7 +466,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);


assertEquals(1, response.getHits().getTotalHits().value);
Expand Down Expand Up @@ -561,7 +561,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);


assertEquals(1, response.getHits().getTotalHits().value);
Expand Down Expand Up @@ -653,7 +653,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);


assertEquals(1, response.getHits().getTotalHits().value);
Expand Down Expand Up @@ -745,7 +745,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);


assertEquals(1, response.getHits().getTotalHits().value);
Expand Down Expand Up @@ -802,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT
/** findings are present but alerts are NOT generated as detection type mentioned in trigger is threat_intel only but finding is from rules*/
Assert.assertEquals(3, getAlertsBody.get("total_alerts"));
}
}
}
1 change: 1 addition & 0 deletions src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ void setDebugLogLevel() throws IOException, InterruptedException {


makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json"));
updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true");
}

protected final List<String> clusterPermissions = List.of(
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ public void testGetAlerts_success() {
List.of(),
Map.of(),
new DataSources(),
true,
TransportIndexDetectorAction.PLUGIN_OWNER_FIELD
),
new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")),
Expand Down Expand Up @@ -129,6 +130,7 @@ public void testGetAlerts_success() {
List.of(),
Map.of(),
new DataSources(),
true,
TransportIndexDetectorAction.PLUGIN_OWNER_FIELD
),
new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")),
Expand Down
22 changes: 11 additions & 11 deletions src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, Inte
}

public void testGetAlerts_noDetector_failure() throws IOException {
// Call GetAlerts API
// Call GetAlerts API
Map<String, String> params = new HashMap<>();
params.put("detector_id", "nonexistent_detector_id");
try {
Expand Down Expand Up @@ -820,7 +820,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
Collections.emptyList());
Detector detector = randomDetectorWithInputsAndTriggers(List.of(input),
List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))
);
);

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));

Expand All @@ -831,7 +831,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);

assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor

Expand Down Expand Up @@ -869,8 +869,8 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
}
}

assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue());
assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue());
assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue());
assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue());

Map<String, String> params = new HashMap<>();
params.put("detector_id", detectorId);
Expand All @@ -894,13 +894,13 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
List<Map<String, Object>> queries = (List<Map<String, Object>>) finding.get("queries");
Set<String> findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet());

// In the case of bucket level monitors, queries will always contain one value
String aggRuleId = findingRuleIds.iterator().next();
List<String> findingDocs = (List<String>) finding.get("related_doc_ids");
// In the case of bucket level monitors, queries will always contain one value
String aggRuleId = findingRuleIds.iterator().next();
List<String> findingDocs = (List<String>) finding.get("related_doc_ids");

if (aggRuleId.equals(sumRuleId)) {
assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs));
}
if (aggRuleId.equals(sumRuleId)) {
assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs));
}
}

assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding));
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1223,7 +1223,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);

assertEquals(1, response.getHits().getTotalHits().value);

Expand Down
Loading

0 comments on commit 18e3ef2

Please sign in to comment.