Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.16] test update detector with aggregation rules #1425

Open
wants to merge 1 commit into
base: 2.16
Choose a base branch
from
Open

[Backport 2.16] test update detector with aggregation rules #1425

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,90 @@ public Script convertToCondition() {
return new Script(condition.toString());
}

public Script convertToConditionForChainedFindings() {
StringBuilder condition = new StringBuilder();

boolean triggerFlag = false;

int size = 0;
if (detectionTypes.contains(RULES_DETECTION_TYPE)) { // trigger should match rules based queries based on conditions
StringBuilder ruleTypeBuilder = new StringBuilder();
size = ruleTypes.size();
for (int idx = 0; idx < size; ++idx) {
ruleTypeBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", ruleTypes.get(idx)));
if (idx < size - 1) {
ruleTypeBuilder.append(" || ");
}
}
if (size > 0) {
condition.append("(").append(ruleTypeBuilder).append(")");
triggerFlag = true;
}

StringBuilder ruleNameBuilder = new StringBuilder();
size = ruleIds.size();
for (int idx = 0; idx < size; ++idx) {
ruleNameBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", ruleIds.get(idx)));
if (idx < size - 1) {
ruleNameBuilder.append(" || ");
}
}
if (size > 0) {
if (triggerFlag) {
condition.append(" && ").append("(").append(ruleNameBuilder).append(")");
} else {
condition.append("(").append(ruleNameBuilder).append(")");
triggerFlag = true;
}
}

StringBuilder ruleSevLevelBuilder = new StringBuilder();
size = ruleSeverityLevels.size();
for (int idx = 0; idx < size; ++idx) {
ruleSevLevelBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", ruleSeverityLevels.get(idx)));
if (idx < size - 1) {
ruleSevLevelBuilder.append(" || ");
}
}

if (size > 0) {
if (triggerFlag) {
condition.append(" && ").append("(").append(ruleSevLevelBuilder).append(")");
} else {
condition.append("(").append(ruleSevLevelBuilder).append(")");
triggerFlag = true;
}
}

StringBuilder tagBuilder = new StringBuilder();
size = tags.size();
for (int idx = 0; idx < size; ++idx) {
tagBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", tags.get(idx)));
if (idx < size - 1) {
ruleSevLevelBuilder.append(" || ");
}
}

if (size > 0) {
if (triggerFlag) {
condition.append(" && ").append("(").append(tagBuilder).append(")");
} else {
condition.append("(").append(tagBuilder).append(")");
}
}
}
if(detectionTypes.contains(THREAT_INTEL_DETECTION_TYPE)) {
StringBuilder threatIntelClauseBuilder = new StringBuilder();
threatIntelClauseBuilder.append(String.format(Locale.getDefault(), "query[tag=%s]", "threat_intel"));
if (condition.length() > 0) {
condition.append(" || ");
}
condition.append("(").append(threatIntelClauseBuilder).append(")");
}

return new Script(condition.toString());
}

public String getId() {
return id;
}
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -854,6 +854,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest(
if(query.getRight().isAggregationRule()) {
Rule rule = query.getRight();
tags.add(rule.getLevel());
tags.add(rule.getId());
tags.add(rule.getCategory());
tags.addAll(rule.getTags().stream().map(Value::getValue).collect(Collectors.toList()));
}
Expand All @@ -879,7 +880,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest(
String name = detectorTrigger.getName();
String severity = detectorTrigger.getSeverity();
List<Action> actions = detectorTrigger.getActions();
Script condition = detectorTrigger.convertToCondition();
Script condition = detectorTrigger.convertToConditionForChainedFindings();

triggers.add(new DocumentLevelTrigger(id, name, severity, actions, condition));
}
Expand Down
119 changes: 119 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -956,6 +956,125 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException
assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue());
}

public void test_detectorWith1AggRuleAndTriggeronRule_updateWithSecondAggRule() throws IOException {
String index = createTestIndex(randomIndex(), windowsIndexMapping());

Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + index + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

Response createMappingResponse = client().performRequest(createMappingRequest);

assertEquals(org.apache.http.HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());

String infoOpCode = "Info";
/** 1st agg rule*/
String sumRuleId = createRule(randomAggregationRule("sum", " > 1", infoOpCode));


List<DetectorRule> detectorRules = List.of(new DetectorRule(sumRuleId));

DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules,
Collections.emptyList());
Detector detector = randomDetectorWithInputsAndTriggers(List.of(input),
List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(sumRuleId), List.of(), List.of(), List.of(), List.of()))
);

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));


String request = "{\n" +
" \"query\" : {\n" +
" \"match_all\":{\n" +
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true);

assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));
Map<String, Object> responseBody = asMap(createResponse);
String detectorId = responseBody.get("_id").toString();
request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + detectorId + "\"\n" +
" }\n" +
" }\n" +
"}";
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);
Map<String, List> updatedDetectorMap = (HashMap<String, List>) (hit.getSourceAsMap().get("detector"));

String workflowId = ((List<String>) (updatedDetectorMap).get("workflow_ids")).get(0);

indexDoc(index, "1", randomDoc(2, 4, infoOpCode));
indexDoc(index, "2", randomDoc(3, 4, infoOpCode));
executeAlertingWorkflow(workflowId, Collections.emptyMap());

Map<String, String> params = new HashMap<>();
params.put("detector_id", detectorId);
Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);

/** assert findings */
assertNotNull(getFindingsBody);
assertEquals(1, getFindingsBody.get("total_findings"));

/**assert alerts */
Map<String, String> params1 = new HashMap<>();
params1.put("detector_id", detectorId);
Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params1, null);
Map<String, Object> getAlertsBody = asMap(getAlertsResponse);

Assert.assertEquals(1, getAlertsBody.get("total_alerts"));
/** 2nd agg rule*/
String sumRuleId2 = createRule(randomAggregationRule("sum", " > 1", infoOpCode));
String sumRuleId3 = createRule(randomAggregationRule("sum", " > 100", infoOpCode));

detectorRules = List.of(new DetectorRule(sumRuleId), new DetectorRule(sumRuleId2));
input = new DetectorInput("updated", List.of("windows"), detectorRules, Collections.emptyList());
Detector updatedDetector = randomDetectorWithInputsAndTriggers(List.of(input),
List.of(new DetectorTrigger("updated1", "test-trigger1", "1", List.of(randomDetectorType()), List.of(sumRuleId2, sumRuleId), List.of(), List.of(), List.of(), List.of()),
new DetectorTrigger("updated2", "test-trigger2", "1", List.of(randomDetectorType()), List.of(sumRuleId2, sumRuleId3), List.of(), List.of(), List.of(), List.of()),
new DetectorTrigger("noAlertsExpected", "test-trigger2", "1", List.of(randomDetectorType()), List.of(sumRuleId3), List.of(), List.of(), List.of(), List.of()))
);
/** update detector and verify chained findings monitor should still exist*/
makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector));
hits = executeSearch(Detector.DETECTORS_INDEX, request);
hit = hits.get(0);
updatedDetectorMap = (HashMap<String, List>) (hit.getSourceAsMap().get("detector"));

assertEquals(3, ((List<String>) (updatedDetectorMap).get("monitor_id")).size());
indexDoc(index, "3", randomDoc(2, 5, infoOpCode));
indexDoc(index, "4", randomDoc(3, 5, infoOpCode));

hits = executeSearch(Detector.DETECTORS_INDEX, request);
hit = hits.get(0);
updatedDetectorMap = (HashMap<String, List>) (hit.getSourceAsMap().get("detector"));

workflowId = ((List<String>) (updatedDetectorMap).get("workflow_ids")).get(0);
executeAlertingWorkflow(workflowId, Collections.emptyMap());

params = new HashMap<>();
params.put("detector_id", detectorId);
getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
getFindingsBody = entityAsMap(getFindingsResponse);

assertNotNull(getFindingsBody);
assertEquals(3, getFindingsBody.get("total_findings"));

params1 = new HashMap<>();
params1.put("detector_id", detectorId);
getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params1, null);
getAlertsBody = asMap(getAlertsResponse);
Assert.assertEquals(3, getAlertsBody.get("total_alerts"));
}

@Ignore
public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, InterruptedException {
updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s");
Expand Down
Loading