diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java index 47dfeed09..7181dda2e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java @@ -376,9 +376,9 @@ private static SearchRequest getSearchRequestForIocType(List indices, St SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); // add the iocs sublist - boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD + ".keyword", iocsSublist)); + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist)); // add ioc type filter - boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD + ".keyword", iocType.toLowerCase(Locale.ROOT))); + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase(Locale.ROOT))); searchRequest.source().query(boolQueryBuilder); return searchRequest; } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java index 1e91ce1f3..77c117784 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -76,8 +76,6 @@ public class TransportListIOCsAction extends HandledTransportAction implements SecureTransportAction { private static final Logger log = LogManager.getLogger(TransportListIOCsAction.class); - public static final String STIX2_IOC_NESTED_PATH = "stix2_ioc."; - private final ClusterService clusterService; private final TransportSearchTIFSourceConfigsAction transportSearchTIFSourceConfigsAction; private final DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService; @@ -184,30 +182,18 @@ private void listIocs(List iocIndices) { // If any of the 'type' options are 'ALL', do not apply 'type' filter if (request.getTypes() != null && request.getTypes().stream().noneMatch(type -> ListIOCsActionRequest.ALL_TYPES_FILTER.equalsIgnoreCase(type))) { for (String type : request.getTypes()) { - boolQueryBuilder.should(QueryBuilders.matchQuery(STIX2_IOC_NESTED_PATH + STIX2IOC.TYPE_FIELD, type)); + boolQueryBuilder.should(QueryBuilders.matchQuery(STIX2IOC.TYPE_FIELD, type)); } boolQueryBuilder.must(typeQueryBuilder); } if (request.getTable().getSearchString() != null && !request.getTable().getSearchString().isEmpty()) { - boolQueryBuilder.must( - QueryBuilders.queryStringQuery(request.getTable().getSearchString()) - .defaultOperator(Operator.OR) -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.ID_FIELD) // Currently not a column in UX table - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.NAME_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.VALUE_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.SEVERITY_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.CREATED_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.MODIFIED_FIELD) -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.DESCRIPTION_FIELD) // Currently not a column in UX table -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.LABELS_FIELD) // Currently not a column in UX table -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.SPEC_VERSION_FIELD) // Currently not a column in UX table - ); + boolQueryBuilder.must(QueryBuilders.queryStringQuery(request.getTable().getSearchString())); } SortBuilder sortBuilder = SortBuilders - .fieldSort(STIX2_IOC_NESTED_PATH + request.getTable().getSortString()) + .fieldSort(request.getTable().getSortString()) .order(SortOrder.fromString(request.getTable().getSortOrder())); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() diff --git a/src/main/resources/mappings/stix2_ioc_mapping.json b/src/main/resources/mappings/stix2_ioc_mapping.json index 8fd505f9a..deaf7c4a1 100644 --- a/src/main/resources/mappings/stix2_ioc_mapping.json +++ b/src/main/resources/mappings/stix2_ioc_mapping.json @@ -3,42 +3,38 @@ "schema_version": 1 }, "properties": { - "stix2_ioc": { - "properties": { - "name": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "value": { - "type": "keyword" - }, - "severity": { - "type": "keyword" - }, - "spec_version": { - "type": "keyword" - }, - "created": { - "type": "date" - }, - "modified": { - "type": "date" - }, - "description": { - "type": "text" - }, - "labels": { - "type": "keyword" - }, - "feed_id": { - "type": "keyword" - }, - "feed_name": { - "type": "keyword" - } - } + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + }, + "spec_version": { + "type": "keyword" + }, + "created": { + "type": "date" + }, + "modified": { + "type": "date" + }, + "description": { + "type": "text" + }, + "labels": { + "type": "keyword" + }, + "feed_id": { + "type": "keyword" + }, + "feed_name": { + "type": "keyword" } } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java index 6207d57b7..63703a201 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -1,165 +1,129 @@ -///* -// * Copyright OpenSearch Contributors -// * SPDX-License-Identifier: Apache-2.0 -// */ -// -//package org.opensearch.securityanalytics.resthandler; -// -//import org.junit.After; -//import org.junit.Assert; -//import org.opensearch.client.Response; -//import org.opensearch.client.WarningFailureException; -//import org.opensearch.common.settings.Settings; -//import org.opensearch.commons.alerting.model.Table; -//import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -//import org.opensearch.securityanalytics.TestHelpers; -//import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -//import org.opensearch.securityanalytics.action.ListIOCsActionResponse; -//import org.opensearch.securityanalytics.commons.model.IOCType; -//import org.opensearch.securityanalytics.commons.model.STIX2; -//import org.opensearch.securityanalytics.model.STIX2IOC; -//import org.opensearch.securityanalytics.services.STIX2IOCFeedStore; -//import org.opensearch.securityanalytics.util.STIX2IOCGenerator; -// -//import java.io.IOException; -//import java.time.Instant; -//import java.util.Arrays; -//import java.util.Collections; -//import java.util.Comparator; -//import java.util.HashMap; -//import java.util.List; -//import java.util.Map; -//import java.util.stream.Collectors; -//import java.util.stream.IntStream; -// -//public class ListIOCsRestApiIT extends SecurityAnalyticsRestTestCase { -// private final String indexMapping = "\"properties\": {\n" + -// " \"stix2_ioc\": {\n" + -// " \"dynamic\": \"false\",\n" + -// " \"properties\": {\n" + -// " \"name\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"type\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"value\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"severity\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"spec_version\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"created\": {\n" + -// " \"type\": \"date\"\n" + -// " },\n" + -// " \"modified\": {\n" + -// " \"type\": \"date\"\n" + -// " },\n" + -// " \"description\": {\n" + -// " \"type\": \"text\"\n" + -// " },\n" + -// " \"labels\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"feed_id\": {\n" + -// " \"type\": \"keyword\"\n" + -// " }\n" + -// " }\n" + -// " }\n" + -// " }"; -// -// private String testFeedSourceConfigId; -// private String indexName; -// ListIOCsActionRequest request; -// -// @After -// public void cleanUp() throws IOException { -//// deleteIndex(indexName); -// -// testFeedSourceConfigId = null; -// indexName = null; -// request = null; -// } -// -// public void test_retrievesIOCs() throws IOException { -// // Create index with mappings -// testFeedSourceConfigId = TestHelpers.randomLowerCaseString(); -// indexName = STIX2IOCFeedStore.getIocIndexAlias(testFeedSourceConfigId); -// -// try { -// createIndex(indexName, Settings.EMPTY, indexMapping); -// } catch (WarningFailureException warningFailureException) { -// // Warns that index names starting with "." will be deprecated, but still creates the index -// } catch (Exception e) { -// fail(String.format("Test index creation failed with error: %s", e)); -// } -// -// // Ingest IOCs -// List iocs = IntStream.range(0, 5) -// .mapToObj(i -> STIX2IOCGenerator.randomIOC()) -// .collect(Collectors.toList()); -// for (STIX2IOC ioc : iocs) { -// indexDoc(indexName, "", STIX2IOCGenerator.toJsonString(ioc)); -// } -// -// request = new ListIOCsActionRequest( -// Arrays.asList(ListIOCsActionRequest.ALL_TYPES_FILTER), -// Arrays.asList(""), new Table( -// "asc", -// "name", -// null, -// iocs.size() + 1, -// 0, -// null) -// ); -// Map params = new HashMap<>(); -// params.put("sortString", request.getTable().getSortString()); -// params.put("size", request.getTable().getSize() + ""); -// params.put("sortOrder", request.getTable().getSortOrder()); -// params.put("searchString", request.getTable().getSearchString() == null ? "" : request.getTable().getSearchString()); -// params.put(ListIOCsActionRequest.TYPE_FIELD, String.join(",", request.getTypes())); -// params.put(STIX2IOC.FEED_ID_FIELD, String.join(",", request.getFeedIds())); -// -// // Retrieve IOCs -// Response response = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(request), params, null); -// Assert.assertEquals(200, response.getStatusLine().getStatusCode()); -// Map respMap = asMap(response); -// -// // Evaluate response -// int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); -// assertEquals(iocs.size(), totalHits); -// -// List> hits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); -// assertEquals(iocs.size(), hits.size()); -// -// // Sort for easy comparison -// iocs.sort(Comparator.comparing(STIX2IOC::getName)); -// hits.sort(Comparator.comparing(hit -> (String) hit.get(STIX2IOC.NAME_FIELD))); -// -// for (int i = 0; i < iocs.size(); i++) { -// Map hit = hits.get(i); -// STIX2IOC newIoc = new STIX2IOC( -// (String) hit.get(STIX2IOC.ID_FIELD), -// (String) hit.get(STIX2IOC.NAME_FIELD), -// IOCType.valueOf((String) hit.get(STIX2IOC.TYPE_FIELD)), -// (String) hit.get(STIX2IOC.VALUE_FIELD), -// (String) hit.get(STIX2IOC.SEVERITY_FIELD), -// Instant.parse((String) hit.get(STIX2IOC.CREATED_FIELD)), -// Instant.parse((String) hit.get(STIX2IOC.MODIFIED_FIELD)), -// (String) hit.get(STIX2IOC.DESCRIPTION_FIELD), -// (List) hit.get(STIX2IOC.LABELS_FIELD), -// (String) hit.get(STIX2IOC.SPEC_VERSION_FIELD), -// (String) hit.get(STIX2IOC.FEED_ID_FIELD), -// (String) hit.get(STIX2IOC.FEED_NAME_FIELD), -// Long.parseLong(String.valueOf(hit.get(STIX2IOC.VERSION_FIELD))) -// // TODO implement DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD check when GetFindings API is added -// ); -//// fixme STIX2IOCGenerator.assertEqualIOCs(iocs.get(i), newIoc); -// } -// } -// -// // TODO: Implement additional tests using various query param combinations -//} +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.STIX2IOCGenerator; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class ListIOCsRestApiIT extends SecurityAnalyticsRestTestCase { + + public void testListIOCsBySearchString() throws IOException { + String searchString = "test-search-string"; + List iocs = List.of( + // The 'name' field matches the searchString + new STIX2IOCDto( + "id1", + searchString, + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ), + // The 'value' field matches the searchString + new STIX2IOCDto( + "id2", + TestHelpers.randomLowerCaseString(), + new IOCType(IOCType.IPV4_TYPE), + searchString, + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ), + // No fields match the searchString + new STIX2IOCDto( + "id3", + "name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_searchstring", + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, iocs), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Retrieve IOCs matching searchString + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("searchString", searchString), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(2, totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(2, iocHits.size()); + + int nameMatch = (int) iocHits.stream().filter((hit) -> Objects.equals(hit.get(STIX2IOC.NAME_FIELD), searchString)).count(); + int valueMatch = (int) iocHits.stream().filter((hit) -> Objects.equals(hit.get(STIX2IOC.VALUE_FIELD), searchString)).count(); + assertEquals(1, nameMatch); + assertEquals(1, valueMatch); + } + + // TODO: Implement additional tests using various query param combinations +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 1085f0041..988fb48d0 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -50,42 +50,38 @@ public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { private final String iocIndexMappings = "\"properties\": {\n" + - " \"stix2_ioc\": {\n" + - " \"properties\": {\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"type\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"value\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"severity\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"spec_version\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"created\": {\n" + - " \"type\": \"date\"\n" + - " },\n" + - " \"modified\": {\n" + - " \"type\": \"date\"\n" + - " },\n" + - " \"description\": {\n" + - " \"type\": \"text\"\n" + - " },\n" + - " \"labels\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"feed_id\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"feed_name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"value\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"severity\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"spec_version\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"created\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"modified\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"description\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"labels\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"feed_id\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"feed_name\": {\n" + + " \"type\": \"keyword\"\n" + " }\n" + " }";