Skip to content

Commit

Permalink
create sap error history index
Browse files Browse the repository at this point in the history 
Signed-off-by: Riya Saxena <[email protected]>
  • Loading branch information
@riysaxen-amzn
riysaxen-amzn committed Feb 16, 2024
1 parent e3362f6 commit a2a430b
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 43 deletions.
9 changes: 1 addition & 8 deletions src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,15 @@
*/
package org.opensearch.securityanalytics.config.monitors;

import java.util.List;
import java.util.stream.Collectors;
import org.opensearch.common.inject.Inject;
import org.opensearch.securityanalytics.logtype.LogTypeService;
import org.opensearch.securityanalytics.model.Detector;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import org.opensearch.securityanalytics.model.LogType;


public class DetectorMonitorConfig {

public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template";
public static final String OPENSEARCH_SAP_ERROR_INDEX = ".opensearch-sap-error-history";

public static String getRuleIndex(String logType) {
return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType);
Expand Down
67 changes: 54 additions & 13 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.cluster.routing.Preference;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.UUIDs;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
Expand Down Expand Up @@ -60,6 +61,7 @@
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.query.BoolQueryBuilder;
Expand Down Expand Up @@ -98,18 +100,11 @@
import org.opensearch.securityanalytics.rules.exceptions.SigmaError;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService;
import org.opensearch.securityanalytics.util.DetectorIndices;
import org.opensearch.securityanalytics.util.DetectorUtils;
import org.opensearch.securityanalytics.util.IndexUtils;
import org.opensearch.securityanalytics.util.MonitorService;
import org.opensearch.securityanalytics.util.RuleIndices;
import org.opensearch.securityanalytics.util.RuleTopicIndices;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.securityanalytics.util.WorkflowService;
import org.opensearch.securityanalytics.util.*;
import org.opensearch.tasks.Task;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.TransportService;

import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
Expand All @@ -118,7 +113,6 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -210,7 +204,6 @@ protected void doExecute(Task task, IndexDetectorRequest request, ActionListener
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN)));
return;
}

checkIndicesAndExecute(task, request, listener, user);
}

Expand Down Expand Up @@ -245,6 +238,11 @@ public void onFailure(Exception e) {
else {
listener.onFailure(SecurityAnalyticsException.wrap(e));
}
try {
createSAPHistoryIndex(request, e.toString());
} catch (IOException ex) {
listener.onFailure(SecurityAnalyticsException.wrap(ex));
}
}
});
}
Expand Down Expand Up @@ -1072,7 +1070,7 @@ void prepareDetectorIndexing() throws Exception {
}
}

void createDetector() {
void createDetector() throws IOException {
Detector detector = request.getDetector();
String ruleTopic = detector.getDetectorType();

Expand All @@ -1087,7 +1085,6 @@ void createDetector() {
log.debug("user from original context is {}", originalContextUser);
request.getDetector().setUser(originalContextUser);


if (!detector.getInputs().isEmpty()) {
try {
ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() {
Expand Down Expand Up @@ -1522,9 +1519,15 @@ private void onOperation(IndexResponse response, Detector detector) {
}

private void onFailures(Exception t) {
log.info("Exception failures while creating detector: {}", t.toString());
if (counter.compareAndSet(false, true)) {
finishHim(null, t);
}
try {
createSAPHistoryIndex(request, t.toString());
} catch (IOException e) {
throw new RuntimeException(e);
}
}

private void finishHim(Detector detector, Exception t) {
Expand Down Expand Up @@ -1573,6 +1576,44 @@ private Map<String, String> mapMonitorIds(List<IndexMonitorResponse> monitorResp
}
}

private void createSAPHistoryIndex(IndexDetectorRequest request, String exception) throws IOException {
Detector detector = request.getDetector();
String ruleTopic = detector.getDetectorType();
String indexName = DetectorMonitorConfig.OPENSEARCH_SAP_ERROR_INDEX;
Instant timestamp = detector.getLastUpdateTime();
String detectorId = detector.getId();
String operation = detectorId.isEmpty() ? "CREATE_DETECTOR" : "UPDATE_DETECTOR";
String user = detector.getUser() == null ? "user" : detector.getUser().getName();

XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
builder.field("detectorId", detectorId);
builder.field("exception", exception);
builder.field("timestamp", timestamp);
builder.field("logType", ruleTopic);
builder.field("operation", operation);
builder.field("user", user);
builder.endObject();
log.info("Index name is: {}", indexName);
IndexRequest indexRequest = new IndexRequest(indexName)
.id(UUIDs.base64UUID())
.source(builder)
.timeout(indexTimeout)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
client.index(indexRequest, new ActionListener<>() {
@Override
public void onResponse(IndexResponse response) {
if (response.status().equals(RestStatus.OK)) {
log.info("Successfully updated the index: {}", indexName);
}
}

@Override
public void onFailure(Exception e) {
log.info("Create error index failed with exception: {}", e);
}
});
}

private void setFilterByEnabled(boolean filterByEnabled) {
this.filterByEnabled = filterByEnabled;
}
Expand Down
76 changes: 54 additions & 22 deletions src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,21 @@
package org.opensearch.securityanalytics.resthandler;

import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.*;

import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicHeader;
import org.junit.Assert;
import org.junit.Ignore;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Request;
import org.opensearch.client.Response;
import org.opensearch.common.settings.Settings;
import org.opensearch.client.ResponseException;
import org.opensearch.common.settings.Settings;
import org.opensearch.commons.alerting.model.IntervalSchedule;
import org.opensearch.commons.alerting.model.Monitor.MonitorType;
import org.opensearch.commons.alerting.model.ScheduledJob;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.search.SearchHit;
Expand All @@ -35,14 +31,9 @@
import org.opensearch.securityanalytics.model.DetectorRule;

import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
import org.opensearch.securityanalytics.model.DetectorTrigger;

import static org.junit.Assert.assertNotNull;
import org.opensearch.test.rest.OpenSearchRestTestCase;
import static org.opensearch.securityanalytics.TestHelpers.*;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE;

Expand Down Expand Up @@ -377,17 +368,19 @@ public void testCreatingADetectorWithMultipleIndices() throws IOException {
Assert.assertEquals(findings.size(), 2);
}

public void testCreatingADetectorWithIndexNotExists() throws IOException {
public void testCreatingADetectorWithIndexNotExists() throws IOException, InterruptedException {
Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())));

try {
makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
} catch (ResponseException ex) {
Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode());
Thread.sleep(30000);
// validate SAP history index if it is created an populated correctly
checkIfSAPErrorIndexExistsAndPopulated("no such index");
}
}

public void testCreatingADetectorWithNonExistingCustomRule() throws IOException {
public void CreatingADetectorWithNonExistingCustomRule() throws IOException, InterruptedException {
String index = createTestIndex(randomIndex(), windowsIndexMapping());

// Execute CreateMappingsAction to add alias mapping for index
Expand All @@ -402,14 +395,17 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException

Response response = client().performRequest(createMappingRequest);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(java.util.UUID.randomUUID().toString())),
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(UUID.randomUUID().toString())),
getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()));
Detector detector = randomDetectorWithInputs(List.of(input));

try {
makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
} catch (ResponseException ex) {
Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode());
Thread.sleep(30000);
// validate SAP history index if it is created an populated correctly
checkIfSAPErrorIndexExistsAndPopulated("Custom Rule Index not found");
}
}

Expand All @@ -418,7 +414,7 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException
* 2. Detector without rules and monitors created successfully
* @throws IOException
*/
public void testCreateDetectorWithoutRules() throws IOException {
public void testCreateDetectorWithoutRules() throws IOException, InterruptedException {
String index = createTestIndex(randomIndex(), windowsIndexMapping());

// Execute CreateMappingsAction to add alias mapping for index
Expand All @@ -442,6 +438,36 @@ public void testCreateDetectorWithoutRules() throws IOException {
} catch (ResponseException ex) {
Assert.assertEquals(400, ex.getResponse().getStatusLine().getStatusCode());
assertTrue(ex.getMessage().contains("Detector cannot be created as no compatible rules were provided"));
Thread.sleep(30000);
// validate SAP history index if it is created an populated correctly
checkIfSAPErrorIndexExistsAndPopulated("no compatible rules");

}
}

private static void checkIfSAPErrorIndexExistsAndPopulated(String exceptionMessage) throws IOException {
String indexName = ".opensearch-sap-error-history";
Boolean searchErrResp = OpenSearchRestTestCase.indexExists(indexName);

// Validate index creation
assertTrue(searchErrResp);
Map<String, Object> searchResponse = OpenSearchRestTestCase.getAsMap("/" + indexName + "/_search");
assertNotNull(searchResponse);
assertTrue(searchResponse.containsKey("hits"));
Map<String, Object> hits = (Map<String, Object>) searchResponse.get("hits");
assertTrue(hits.containsKey("hits"));
List<Map<String, Object>> hitList = (List<Map<String, Object>>) hits.get("hits");
assertTrue(hitList.size() > 0);

// Iterate through each hit
for (Map<String, Object> hit : hitList) {
Map<String, Object> source = (Map<String, Object>) hit.get("_source");
assertNotNull(source);

// Validate the "exception" field in each hit
assertTrue(source.containsKey("exception"));
String exception = (String) source.get("exception");
assertTrue(exception.contains(exceptionMessage));
}
}

Expand Down Expand Up @@ -857,7 +883,7 @@ public void testUpdateADetector() throws IOException {
Assert.assertEquals(6, response.getHits().getTotalHits().value);
}

public void testUpdateANonExistingDetector() throws IOException {
public void testUpdateANonExistingDetector() throws IOException, InterruptedException {
String index = createTestIndex(randomIndex(), windowsIndexMapping());

// Execute CreateMappingsAction to add alias mapping for index
Expand All @@ -875,21 +901,27 @@ public void testUpdateANonExistingDetector() throws IOException {
Detector updatedDetector = randomDetectorWithInputs(List.of(input));

try {
makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector));
makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector));
} catch (ResponseException ex) {
Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode());
Thread.sleep(30000);
// validate SAP history index if it is created an populated correctly
checkIfSAPErrorIndexExistsAndPopulated("not found");
}
}

public void testUpdateADetectorWithIndexNotExists() throws IOException {
public void testUpdateADetectorWithIndexNotExists() throws IOException, InterruptedException {
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(),
getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()));
Detector updatedDetector = randomDetectorWithInputs(List.of(input));

try {
makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector));
makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector));
} catch (ResponseException ex) {
Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode());
Thread.sleep(30000);
// validate SAP history index if it is created an populated correctly
checkIfSAPErrorIndexExistsAndPopulated("no such index");
}
}

Expand Down Expand Up @@ -1279,7 +1311,7 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro

public void testDeletingANonExistingDetector() throws IOException {
try {
makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), null);
makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), null);
} catch (ResponseException ex) {
Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode());
}
Expand Down

0 comments on commit a2a430b

Please sign in to comment.