From bf831f390115ede61db6ecb5c1c337a2284df760 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:02:47 -0700 Subject: [PATCH] fix for detectors with sigma aggregation rules (#1372) (#1375) (cherry picked from commit 6f543b508d7cde8a232b29b7e1786ab2cbd9c13c) Signed-off-by: Subhobrata Dey Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../mapper/IndexTemplateManager.java | 15 ++- .../TransportIndexDetectorAction.java | 8 +- .../SecurityAnalyticsRestTestCase.java | 39 ++++++- .../securityanalytics/TestHelpers.java | 4 +- .../resthandler/DetectorMonitorRestApiIT.java | 108 +++++++++++++++++- 5 files changed, 166 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java b/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java index 4bc41e4ba..2f3c10cdb 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java @@ -115,13 +115,22 @@ public void upsertIndexTemplateWithAliasMappings( upsertComponentTemplateStepListener.whenComplete( acknowledgedResponse -> { - // Find template which matches input index best + // Find template which matches input index best. starts by directly matching with input index and + // if not found matches with current write index. String templateName = MetadataIndexTemplateService.findV2Template( state.metadata(), normalizeIndexName(indexName), false ); + if (templateName == null) { + templateName = + MetadataIndexTemplateService.findV2Template( + state.metadata(), + normalizeIndexName(cin), + false + ); + } if (templateName == null) { // If we find conflicting templates(regardless of priority) and that template was created by us, @@ -181,8 +190,8 @@ public void upsertIndexTemplateWithAliasMappings( template = state.metadata().templatesV2().get(templateName); if (template.composedOf().contains(componentName) == false) { List newComposedOf = new ArrayList<>(template.composedOf()); - List indexPatterns = List.of(computeIndexPattern(indexName)); - ; + List indexPatterns = new ArrayList<>(template.indexPatterns()); + indexPatterns.add(computeIndexPattern(indexName)); newComposedOf.add(componentName); try { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index f415d0f2a..40918c157 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -863,12 +863,16 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( } } tags.removeIf(Objects::isNull); + + // if queryFieldNames is not passed, alerting doc-level monitor fetches entire log doc. + List queryFieldNames = List.of("_id"); DocLevelQuery docLevelQuery = new DocLevelQuery( monitorName, monitorName + "doc", Collections.emptyList(), actualQuery, - new ArrayList<>(tags) + new ArrayList<>(tags), + queryFieldNames ); docLevelQueries.add(docLevelQuery); @@ -1037,6 +1041,8 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { boolQueryBuilder.must(timeRangeFilter); searchSourceBuilder.query(boolQueryBuilder); } + // query hits are not needed from this query part for aggregations. + searchSourceBuilder.size(0); List bucketLevelMonitorInputs = new ArrayList<>(); bucketLevelMonitorInputs.add(new SearchInput(indices, searchSourceBuilder)); diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 26ffe4344..8bd62b34e 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -1729,7 +1729,8 @@ protected void createComposableIndexTemplate(String templateName, List i Collectors.joining(",", "\"", "\"")) + "]," + (componentTemplateName == null ? ("\"template\": {\"mappings\": {" + mappings + "}},") : "") + - (componentTemplateName != null ? ("\"composed_of\": [\"" + componentTemplateName + "\"],") : "") + + (componentTemplateName != null ? ("\"composed_of\": [\"" + componentTemplateName + "\"],\"template\": {" + + "\"settings\": {\"index\":{\"mapping\":{\"total_fields\":{\"limit\":\"5000\"}},\"number_of_shards\":\"18\",\"number_of_replicas\":\"1\"}}},") : "") + "\"priority\":" + priority + "}"; Response response = makeRequest( @@ -1810,6 +1811,14 @@ protected void createDatastreamAPI(String datastreamName) throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + protected void createIndexAliasApi(String indexAlias, String indexName) throws IOException { + Request request = new Request("POST", "_aliases"); + request.setJsonEntity("{\"actions\":[{\"add\":{\"index\":\"" + indexName + "\",\"alias\":\"" + indexAlias + "\", " + + "\"is_write_index\": true}}]}"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + protected void deleteDatastreamAPI(String datastreamName) throws IOException { Request request = new Request("DELETE", "_data_stream/" + datastreamName); @@ -1852,6 +1861,34 @@ protected void createSampleDatastream(String datastreamName, String mappings, bo createDatastreamAPI(datastreamName); } + protected void createSampleIndexTemplate(String indexPattern, String mappings, boolean useComponentTemplate) throws IOException { + String indexName = indexPattern.substring(0, indexPattern.length() - 1); + String componentTemplateMappings = "\"properties\": {" + + " \"netflow.destination_transport_port\":{ \"type\": \"long\" }," + + " \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" + + "}"; + + if (mappings != null) { + componentTemplateMappings = mappings; + } + + if (useComponentTemplate) { + // Setup index_template + createComponentTemplateWithMappings( + "my_ds_component_template-" + indexName, + componentTemplateMappings + ); + } + createComposableIndexTemplate( + "my_index_template_ds-" + indexName, + List.of(indexPattern), + useComponentTemplate ? "my_ds_component_template-" + indexName : null, + mappings, + false, + 2 + ); + } + protected void restoreAlertsFindingsIMSettings() throws IOException { updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "720m"); updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "100000"); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 4a7fffd06..2d826eba1 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1159,7 +1159,7 @@ public static String randomCloudtrailAggrRuleWithDotFields() { " - lambda.amazonaws.com\n" + " api.operation: \n" + " - Invoke\n" + - " timeframe: 20m\n" + + " timeframe: 1m\n" + " tags:\n" + " - attack.privilege_escalation\n" + " - attack.t1078"; @@ -2308,7 +2308,7 @@ public static String randomCloudtrailOcsfDoc() { " },\n" + " \"status\": \"Success\",\n" + " \"status_id\": 1,\n" + - " \"time\": 1702952105000,\n" + + " \"time\": " + System.currentTimeMillis() + ",\n" + " \"type_name\": \"Account Change: Detach Policy\",\n" + " \"type_uid\": 300108,\n" + " \"unmapped\": {\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index f58c91831..9ac49abbf 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -12,6 +12,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.Monitor.MonitorType; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -2063,7 +2064,10 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithDotFields() throws IOExc @SuppressWarnings("unchecked") public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOException { - String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + String index = "cloudtrail"; + String indexAlias = "test_alias"; + + createIndex(index, Settings.EMPTY, cloudtrailOcsfMappings(), "\"" + indexAlias + "\":{\"is_write_index\": true}"); // Execute CreateMappingsAction to add alias mapping for index Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -2153,6 +2157,108 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOExc assertEquals(1, getFindingsBody.get("total_findings")); } + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithRolloverIndexAliases() throws IOException, InterruptedException { + createSampleIndexTemplate("cloudtrail*", cloudtrailOcsfMappings(), true); + String index = createTestIndex("cloudtrail-000001", ""); + createIndexAliasApi("ocsf_ct", "cloudtrail-000001"); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"ocsf_ct\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"timestamp\": {\n" + + " \"path\": \"time\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithDotFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of("ocsf_ct"), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc("ocsf_ct", "1", randomCloudtrailOcsfDoc()); + indexDoc("ocsf_ct", "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + + doRollover("ocsf_ct"); + Thread.sleep(90000); + + indexDoc("ocsf_ct", "4", randomCloudtrailOcsfDoc()); + indexDoc("ocsf_ct", "5", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + params = new HashMap<>(); + params.put("detector_id", createdId); + getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(2, getFindingsBody.get("total_findings")); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum();