diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 459f523b7..7cb8d90af 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -4,22 +4,15 @@ */ package org.opensearch.securityanalytics.config.monitors; -import java.util.List; -import java.util.stream.Collectors; -import org.opensearch.common.inject.Inject; -import org.opensearch.securityanalytics.logtype.LogTypeService; -import org.opensearch.securityanalytics.model.Detector; - -import java.util.Arrays; import java.util.HashMap; import java.util.Locale; import java.util.Map; -import org.opensearch.securityanalytics.model.LogType; public class DetectorMonitorConfig { public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; + public static final String OPENSEARCH_SAP_ERROR_INDEX = ".opensearch-sap-error-history"; public static String getRuleIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index e6dea9947..7d5df810f 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -31,6 +31,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.UUIDs; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -60,6 +61,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.BoolQueryBuilder; @@ -98,18 +100,11 @@ import org.opensearch.securityanalytics.rules.exceptions.SigmaError; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.util.DetectorIndices; -import org.opensearch.securityanalytics.util.DetectorUtils; -import org.opensearch.securityanalytics.util.IndexUtils; -import org.opensearch.securityanalytics.util.MonitorService; -import org.opensearch.securityanalytics.util.RuleIndices; -import org.opensearch.securityanalytics.util.RuleTopicIndices; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import org.opensearch.securityanalytics.util.WorkflowService; +import org.opensearch.securityanalytics.util.*; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; - +import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -118,7 +113,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -210,7 +204,6 @@ protected void doExecute(Task task, IndexDetectorRequest request, ActionListener listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); return; } - checkIndicesAndExecute(task, request, listener, user); } @@ -245,6 +238,11 @@ public void onFailure(Exception e) { else { listener.onFailure(SecurityAnalyticsException.wrap(e)); } + try { + createSAPHistoryIndex(request, e.toString()); + } catch (IOException ex) { + listener.onFailure(SecurityAnalyticsException.wrap(ex)); + } } }); } @@ -1072,7 +1070,7 @@ void prepareDetectorIndexing() throws Exception { } } - void createDetector() { + void createDetector() throws IOException { Detector detector = request.getDetector(); String ruleTopic = detector.getDetectorType(); @@ -1087,7 +1085,6 @@ void createDetector() { log.debug("user from original context is {}", originalContextUser); request.getDetector().setUser(originalContextUser); - if (!detector.getInputs().isEmpty()) { try { ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() { @@ -1522,9 +1519,15 @@ private void onOperation(IndexResponse response, Detector detector) { } private void onFailures(Exception t) { + log.info("Exception failures while creating detector: {}", t.toString()); if (counter.compareAndSet(false, true)) { finishHim(null, t); } + try { + createSAPHistoryIndex(request, t.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } } private void finishHim(Detector detector, Exception t) { @@ -1573,6 +1576,44 @@ private Map mapMonitorIds(List monitorResp } } + private void createSAPHistoryIndex(IndexDetectorRequest request, String exception) throws IOException { + Detector detector = request.getDetector(); + String ruleTopic = detector.getDetectorType(); + String indexName = DetectorMonitorConfig.OPENSEARCH_SAP_ERROR_INDEX; + Instant timestamp = detector.getLastUpdateTime(); + String detectorId = detector.getId(); + String operation = detectorId.isEmpty() ? "CREATE_DETECTOR" : "UPDATE_DETECTOR"; + String user = detector.getUser() == null ? "user" : detector.getUser().getName(); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder.field("detectorId", detectorId); + builder.field("exception", exception); + builder.field("timestamp", timestamp); + builder.field("logType", ruleTopic); + builder.field("operation", operation); + builder.field("user", user); + builder.endObject(); + log.info("Index name is: {}", indexName); + IndexRequest indexRequest = new IndexRequest(indexName) + .id(UUIDs.base64UUID()) + .source(builder) + .timeout(indexTimeout) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status().equals(RestStatus.OK)) { + log.info("Successfully updated the index: {}", indexName); + } + } + + @Override + public void onFailure(Exception e) { + log.info("Create error index failed with exception: {}", e); + } + }); + } + private void setFilterByEnabled(boolean filterByEnabled) { this.filterByEnabled = filterByEnabled; } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 541925dd5..1975cb415 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -5,9 +5,7 @@ package org.opensearch.securityanalytics.resthandler; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; +import java.util.*; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.HttpEntity; @@ -15,15 +13,13 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; -import org.junit.Ignore; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.common.settings.Settings; import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor.MonitorType; -import org.opensearch.commons.alerting.model.ScheduledJob; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.search.SearchHit; @@ -35,14 +31,9 @@ import org.opensearch.securityanalytics.model.DetectorRule; import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; import java.util.stream.Collectors; import org.opensearch.securityanalytics.model.DetectorTrigger; - -import static org.junit.Assert.assertNotNull; +import org.opensearch.test.rest.OpenSearchRestTestCase; import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; @@ -377,17 +368,19 @@ public void testCreatingADetectorWithMultipleIndices() throws IOException { Assert.assertEquals(findings.size(), 2); } - public void testCreatingADetectorWithIndexNotExists() throws IOException { + public void testCreatingADetectorWithIndexNotExists() throws IOException, InterruptedException { Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); try { makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("no such index"); } } - - public void testCreatingADetectorWithNonExistingCustomRule() throws IOException { +public void CreatingADetectorWithNonExistingCustomRule() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -402,7 +395,7 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException Response response = client().performRequest(createMappingRequest); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(java.util.UUID.randomUUID().toString())), + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(UUID.randomUUID().toString())), getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input)); @@ -410,6 +403,9 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("Custom Rule Index not found"); } } @@ -418,7 +414,7 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException * 2. Detector without rules and monitors created successfully * @throws IOException */ - public void testCreateDetectorWithoutRules() throws IOException { + public void testCreateDetectorWithoutRules() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -442,6 +438,36 @@ public void testCreateDetectorWithoutRules() throws IOException { } catch (ResponseException ex) { Assert.assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); assertTrue(ex.getMessage().contains("Detector cannot be created as no compatible rules were provided")); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("no compatible rules"); + + } + } + + private static void checkIfSAPErrorIndexExistsAndPopulated(String exceptionMessage) throws IOException { + String indexName = ".opensearch-sap-error-history"; + Boolean searchErrResp = OpenSearchRestTestCase.indexExists(indexName); + + // Validate index creation + assertTrue(searchErrResp); + Map searchResponse = OpenSearchRestTestCase.getAsMap("/" + indexName + "/_search"); + assertNotNull(searchResponse); + assertTrue(searchResponse.containsKey("hits")); + Map hits = (Map) searchResponse.get("hits"); + assertTrue(hits.containsKey("hits")); + List> hitList = (List>) hits.get("hits"); + assertTrue(hitList.size() > 0); + + // Iterate through each hit + for (Map hit : hitList) { + Map source = (Map) hit.get("_source"); + assertNotNull(source); + + // Validate the "exception" field in each hit + assertTrue(source.containsKey("exception")); + String exception = (String) source.get("exception"); + assertTrue(exception.contains(exceptionMessage)); } } @@ -857,7 +883,7 @@ public void testUpdateADetector() throws IOException { Assert.assertEquals(6, response.getHits().getTotalHits().value); } - public void testUpdateANonExistingDetector() throws IOException { + public void testUpdateANonExistingDetector() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -875,21 +901,27 @@ public void testUpdateANonExistingDetector() throws IOException { Detector updatedDetector = randomDetectorWithInputs(List.of(input)); try { - makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("not found"); } } - public void testUpdateADetectorWithIndexNotExists() throws IOException { + public void testUpdateADetectorWithIndexNotExists() throws IOException, InterruptedException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); Detector updatedDetector = randomDetectorWithInputs(List.of(input)); try { - makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("no such index"); } } @@ -1279,7 +1311,7 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro public void testDeletingANonExistingDetector() throws IOException { try { - makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), null); + makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), null); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); }