Skip to content

Commit

Permalink
Backport opensearch-project#1335 to 2.x (opensearch-project#1339) (op…
Browse files Browse the repository at this point in the history 
…ensearch-project#1340)

* Add null check while adding fetched iocs into per-indicator-type map (opensearch-project#1335)

* add null check while adding fetched iocs into per-indicator-type map

Signed-off-by: Surya Sashank Nistala <[email protected]>

* adds tests verifying monitor configured on multiple indicator types

Signed-off-by: Surya Sashank Nistala <[email protected]>

---------

Signed-off-by: Surya Sashank Nistala <[email protected]>

* fix constructor for 2.x

Signed-off-by: Surya Sashank Nistala <[email protected]>

---------

Signed-off-by: Surya Sashank Nistala <[email protected]>
(cherry picked from commit 01d9011)

Co-authored-by: Surya Sashank Nistala <[email protected]>
  • Loading branch information
@opensearch-trigger-bot @eirsep
opensearch-trigger-bot[bot] and eirsep authored Oct 3, 2024
1 parent 5cb2dd4 commit 58e0dd9
Show file tree
Hide file tree
Showing 2 changed files with 195 additions and 21 deletions.
38 changes: 19 additions & 19 deletions ...ain/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,28 +128,28 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs(
for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) {
String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase();
String concreteIndex = getIndexName(datum);
if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)
&& false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty()
) {
if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)) {
// if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc.
String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0);
List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
for (String field : fields) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>());
iocs.addAll(vals);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
List<String> fieldsConfiguredInMonitorForCurrentIndex = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
if(fieldsConfiguredInMonitorForCurrentIndex != null && false == fieldsConfiguredInMonitorForCurrentIndex.isEmpty()) {
for (String field : fieldsConfiguredInMonitorForCurrentIndex) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
}
}
}
}
Expand Down
178 changes: 176 additions & 2 deletions src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers;
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS;
import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH;

public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase {
Expand Down Expand Up @@ -81,6 +80,71 @@ public String indexSourceConfigsAndIocs(List<String> iocVals) throws IOException
return indexTifSourceConfig(testIocDtos);
}

public String indexSourceConfigsAndIocs(List<String> ipVals, List<String> hashVals, List<String> domainVals) throws IOException {
testIocDtos = new ArrayList<>();
for (int i1 = 0; i1 < ipVals.size(); i1++) {
// create IOCs
STIX2IOCDto stix2IOCDto = new STIX2IOCDto(
"id" + randomAlphaOfLength(3),
"random",
new IOCType(IOCType.IPV4_TYPE),
ipVals.get(i1),
"",
Instant.now(),
Instant.now(),
"",
emptyList(),
"spec",
"configId",
"",
1L
);

testIocDtos.add(stix2IOCDto);
}
for (int i1 = 0; i1 < hashVals.size(); i1++) {
// create IOCs
STIX2IOCDto stix2IOCDto = new STIX2IOCDto(
"id" + randomAlphaOfLength(3),
"random",
new IOCType(IOCType.HASHES_TYPE),
hashVals.get(i1),
"",
Instant.now(),
Instant.now(),
"",
emptyList(),
"spec",
"configId",
"",
1L
);

testIocDtos.add(stix2IOCDto);
}
for (int i1 = 0; i1 < domainVals.size(); i1++) {
// create IOCs
STIX2IOCDto stix2IOCDto = new STIX2IOCDto(
"id" + randomAlphaOfLength(3),
"random",
new IOCType(IOCType.DOMAIN_NAME_TYPE),
domainVals.get(i1),
"",
Instant.now(),
Instant.now(),
"",
emptyList(),
"spec",
"configId",
"",
1L
);

testIocDtos.add(stix2IOCDto);
}
return indexTifSourceConfig(testIocDtos);
}

private String indexTifSourceConfig(List<STIX2IOCDto> testIocDtos) throws IOException {
SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto(
"configId",
Expand All @@ -100,7 +164,7 @@ private String indexTifSourceConfig(List<STIX2IOCDto> testIocDtos) throws IOExce
null,
null,
false,
List.of(IOCType.IPV4_TYPE),
List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE, IOCType.DOMAIN_NAME_TYPE),
true
);

Expand Down Expand Up @@ -251,6 +315,96 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException {
assertEquals(totalHitsVal.intValue(), 0);
}



public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonitor() throws IOException {
updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1");
Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Map.of(), null);
Map<String, Object> responseAsMap = responseAsMap(iocFindingsResponse);
Assert.assertEquals(0, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());
List<String> ipVals = List.of("ip1", "ip2");
List<String> hashVals = List.of("h1", "h2");
List<String> domainVals = List.of("d1", "d2");
String createdId = indexSourceConfigsAndIocs(ipVals, hashVals, domainVals);

String ipIndex = "ipAlias";
createTestAlias(ipIndex, 1, true);
String hashIndex = "hashAlias";
createTestAlias(hashIndex, 1, true);
String domainIndex = "domainAlias";
createTestAlias(domainIndex, 1, true);


/**create monitor */
ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(ipIndex, hashIndex, domainIndex);
Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
Assert.assertEquals(201, response.getStatusLine().getStatusCode());
Map<String, Object> responseBody = asMap(response);

try {
makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
fail();
} catch (Exception e) {
/** creating a second threat intel monitor should fail*/
assertTrue(e.getMessage().contains("already exists"));
}

final String monitorId = responseBody.get("id").toString();
Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId);
Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode());

Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId);
Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
int i = 1;
for (String val : ipVals) {
String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(ipIndex, "" + i++, doc);
} catch (IOException e) {
fail();
}
}
for (String val : hashVals) {
String doc = String.format("{\"hash\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(hashIndex, "" + i++, doc);
} catch (IOException e) {
fail();
}
}
for (String val : domainVals) {
String doc = String.format("{\"domain\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(domainIndex, "" + i++, doc);
} catch (IOException e) {
fail();
}
}

executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Map<String, Object> executeResults = entityAsMap(executeResponse);

String matchAllRequest = getMatchAllRequest();
Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));
Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
HashMap<String, Object> hits = (HashMap<String, Object>) asMap(searchMonitorResponse).get("hits");
HashMap<String, Object> totalHits = (HashMap<String, Object>) hits.get("total");
Integer totalHitsVal = (Integer) totalHits.get("value");
assertEquals(totalHitsVal.intValue(), 1);
makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));

iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Map.of(), null);
responseAsMap = responseAsMap(iocFindingsResponse);
Assert.assertEquals(6, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());

//alerts
List<SearchHit> searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest);
Assert.assertEquals(6, searchHits.size());
}

public void testCreateThreatIntelMonitor() throws IOException {
updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1");
Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Expand Down Expand Up @@ -572,6 +726,26 @@ public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) {
List.of(t1, t2, t3, t4));
}

public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(String ipIndex, String hashIndex, String domainIndex) {
ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(ipIndex, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity");
ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity");
ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity");
ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(ipIndex), emptyList(), emptyList(), "indexmatch", null, "severity");

return new ThreatIntelMonitorDto(
Monitor.NO_ID,
randomAlphaOfLength(10),
List.of(
new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(ipIndex, List.of("ip"))),
new PerIocTypeScanInputDto(IOCType.HASHES_TYPE, Map.of(hashIndex, List.of("hash"))),
new PerIocTypeScanInputDto(IOCType.DOMAIN_NAME_TYPE, Map.of(domainIndex, List.of("domain")))
),
new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()),
false,
null,
List.of(t1, t2, t3, t4));
}

@Override
protected boolean preserveIndicesUponCompletion() {
return false;
Expand Down

0 comments on commit 58e0dd9

Please sign in to comment.