diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index f48ab39ae..a3164bb7c 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -67,6 +67,7 @@ import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.IndexRuleAction; import org.opensearch.securityanalytics.action.ListCorrelationsAction; +import org.opensearch.securityanalytics.action.ListIOCsAction; import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction; import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; import org.opensearch.securityanalytics.action.SearchDetectorAction; @@ -104,6 +105,7 @@ import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction; import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction; import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction; +import org.opensearch.securityanalytics.resthandler.RestListIOCsAction; import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction; import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction; import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction; @@ -167,6 +169,7 @@ import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction; import org.opensearch.securityanalytics.transport.TransportIndexRuleAction; import org.opensearch.securityanalytics.transport.TransportListCorrelationAction; +import org.opensearch.securityanalytics.transport.TransportListIOCsAction; import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction; import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction; import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction; @@ -212,6 +215,7 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel"; public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source"; public static final String THREAT_INTEL_MONITOR_URI = PLUGINS_BASE_URI + "/threat_intel/monitor"; + public static final String LIST_IOCS_URI = PLUGINS_BASE_URI + "/iocs/list"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; @@ -336,7 +340,8 @@ public List getRestHandlers(Settings settings, new RestSearchTIFSourceConfigsAction(), new RestIndexThreatIntelMonitorAction(), new RestDeleteThreatIntelMonitorAction(), - new RestSearchThreatIntelMonitorAction() + new RestSearchThreatIntelMonitorAction(), + new RestListIOCsAction() ); } @@ -479,7 +484,8 @@ public List> getSettings() { new ActionHandler<>(SAGetTIFSourceConfigAction.INSTANCE, TransportGetTIFSourceConfigAction.class), new ActionHandler<>(SADeleteTIFSourceConfigAction.INSTANCE, TransportDeleteTIFSourceConfigAction.class), new ActionHandler<>(SASearchTIFSourceConfigsAction.INSTANCE, TransportSearchTIFSourceConfigsAction.class), - new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class) + new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class), + new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java new file mode 100644 index 000000000..0e7e807b1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionType; + +public class ListIOCsAction extends ActionType { + public static final ListIOCsAction INSTANCE = new ListIOCsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/iocs/list"; + + public ListIOCsAction() { + super(NAME, ListIOCsActionResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java b/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java new file mode 100644 index 000000000..dc1b1ef18 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java @@ -0,0 +1,121 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.securityanalytics.commons.model.IOCType; + +import java.io.IOException; +import java.util.Locale; + +public class ListIOCsActionRequest extends ActionRequest { + public static String START_INDEX_FIELD = "start"; + public static String SIZE_FIELD = "size"; + public static String SORT_ORDER_FIELD = "sort_order"; + public static String SORT_STRING_FIELD = "sort_string"; + public static String SEARCH_FIELD = "search"; + public static String TYPE_FIELD = "type"; + public static String ALL_TYPES_FILTER = "ALL"; + + private int startIndex; + private int size; + private SortOrder sortOrder; + private String sortString; + + private String search; + private String type; + private String feedId; + + public ListIOCsActionRequest(int startIndex, int size, String sortOrder, String sortString, String search, String type, String feedId) { + super(); + this.startIndex = startIndex; + this.size = size; + this.sortOrder = SortOrder.valueOf(sortOrder.toLowerCase(Locale.ROOT)); + this.sortString = sortString; + this.search = search; + this.type = type.toLowerCase(Locale.ROOT); + this.feedId = feedId; + } + + public ListIOCsActionRequest(StreamInput sin) throws IOException { + this( + sin.readInt(), // startIndex + sin.readInt(), // size + sin.readString(), // sortOrder + sin.readString(), // sortString + sin.readOptionalString(), // search + sin.readOptionalString(), // type + sin.readOptionalString() //feedId + ); + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(startIndex); + out.writeInt(size); + out.writeEnum(sortOrder); + out.writeString(sortString); + out.writeOptionalString(search); + out.writeOptionalString(type); + out.writeOptionalString(feedId); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (startIndex < 0) { + validationException = ValidateActions + .addValidationError(String.format("[%s] param cannot be a negative number.", START_INDEX_FIELD), validationException); + } else if (size < 0 || size > 10000) { + validationException = ValidateActions + .addValidationError(String.format("[%s] param must be between 0 and 10,000.", SIZE_FIELD), validationException); + } else if (!ALL_TYPES_FILTER.equalsIgnoreCase(type)) { + try { + IOCType.valueOf(type); + } catch (Exception e) { + validationException = ValidateActions + .addValidationError(String.format("Unrecognized [%s] param.", TYPE_FIELD), validationException); + } + } + return validationException; + } + + public int getStartIndex() { + return startIndex; + } + + public int getSize() { + return size; + } + + public SortOrder getSortOrder() { + return sortOrder; + } + + public String getSortString() { + return sortString; + } + + public String getSearch() { + return search; + } + + public String getType() { + return type; + } + + public String getFeedId() { + return feedId; + } + + public enum SortOrder { + asc, + dsc + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java b/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java new file mode 100644 index 000000000..8ca77f088 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.STIX2IOCDto; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class ListIOCsActionResponse extends ActionResponse implements ToXContentObject { + public static String TOTAL_HITS_FIELD = "total"; + public static String HITS_FIELD = "iocs"; + + public static ListIOCsActionResponse EMPTY_RESPONSE = new ListIOCsActionResponse(0, Collections.emptyList()); + + private long totalHits; + private List hits; + + public ListIOCsActionResponse(long totalHits, List hits) { + super(); + this.totalHits = totalHits; + this.hits = hits; + } + + public ListIOCsActionResponse(StreamInput sin) throws IOException { + this(sin.readInt(), sin.readList(STIX2IOCDto::new)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(totalHits); + out.writeList(hits); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(TOTAL_HITS_FIELD, totalHits) + .field(HITS_FIELD, hits) + .endObject(); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java index aa0d8d7e0..8d99394a8 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; @@ -161,7 +161,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws Instant created = null; Instant modified = null; String description = null; - List labels = Collections.emptyList(); + List labels = new ArrayList<>(); String feedId = null; String specVersion = null; @@ -190,7 +190,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { created = null; } else if (xcp.currentToken().isValue()) { - created = Instant.ofEpochMilli(xcp.longValue()); + created = Instant.parse(xcp.text()); } else { XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); created = null; @@ -200,7 +200,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { modified = null; } else if (xcp.currentToken().isValue()) { - modified = Instant.ofEpochMilli(xcp.longValue()); + modified = Instant.parse(xcp.text()); } else { XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); modified = null; diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java index b82899ce9..7ffe5a007 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java @@ -19,7 +19,7 @@ import java.io.IOException; import java.time.Instant; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -42,7 +42,7 @@ public class STIX2IOCDto implements Writeable, ToXContentObject { private String specVersion; private long version; - // No arguments contructor needed for parsing from S3 + // No arguments constructor needed for parsing from S3 public STIX2IOCDto() {} public STIX2IOCDto( @@ -148,7 +148,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr Instant created = null; Instant modified = null; String description = null; - List labels = Collections.emptyList(); + List labels = new ArrayList<>(); String feedId = null; String specVersion = null; @@ -162,7 +162,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr name = xcp.text(); break; case STIX2.TYPE_FIELD: - type = IOCType.valueOf(xcp.text().toUpperCase(Locale.ROOT)); + type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT)); break; case STIX2.VALUE_FIELD: value = xcp.text(); @@ -174,7 +174,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { created = null; } else if (xcp.currentToken().isValue()) { - created = Instant.ofEpochMilli(xcp.longValue()); + created = Instant.parse(xcp.text()); } else { XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); created = null; @@ -184,7 +184,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { modified = null; } else if (xcp.currentToken().isValue()) { - modified = Instant.ofEpochMilli(xcp.longValue()); + modified = Instant.parse(xcp.text()); } else { XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); modified = null; diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java new file mode 100644 index 000000000..5d6f97b70 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.ListIOCsAction; +import org.opensearch.securityanalytics.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.model.STIX2IOC; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class RestListIOCsAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestListIOCsAction.class); + + public String getName() { + return "list_iocs_action"; + } + + public List routes() { + return List.of( + new Route(RestRequest.Method.GET, SecurityAnalyticsPlugin.LIST_IOCS_URI) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.ROOT, "%s %s", request.method(), SecurityAnalyticsPlugin.LIST_IOCS_URI)); + + int startIndex = request.paramAsInt(ListIOCsActionRequest.START_INDEX_FIELD, 0); + int size = request.paramAsInt(ListIOCsActionRequest.SIZE_FIELD, 10); + String sortOrder = request.param(ListIOCsActionRequest.SORT_ORDER_FIELD, ListIOCsActionRequest.SortOrder.asc.toString()); + String sortString = request.param(ListIOCsActionRequest.SORT_STRING_FIELD, STIX2.NAME_FIELD); + String search = request.param(ListIOCsActionRequest.SEARCH_FIELD, ""); + String type = request.param(ListIOCsActionRequest.TYPE_FIELD, ListIOCsActionRequest.ALL_TYPES_FILTER); + String feedId = request.param(STIX2IOC.FEED_ID_FIELD, ""); + + ListIOCsActionRequest listRequest = new ListIOCsActionRequest(startIndex, size, sortOrder, sortString, search, type, feedId); + + return channel -> client.execute(ListIOCsAction.INSTANCE, listRequest, new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(ListIOCsActionResponse response) throws Exception { + return new BytesRestResponse(RestStatus.OK, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + } + }); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java new file mode 100644 index 000000000..7737b0c08 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -0,0 +1,209 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.ActionRunnable; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.routing.Preference; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.action.ListIOCsAction; +import org.opensearch.securityanalytics.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.services.STIX2IOCFeedStore; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public class TransportListIOCsAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportListIOCsAction.class); + + public static final String STIX2_IOC_NESTED_PATH = "stix2_ioc."; + + private final Client client; + private final NamedXContentRegistry xContentRegistry; + private final ThreadPool threadPool; + + @Inject + public TransportListIOCsAction( + TransportService transportService, + Client client, + NamedXContentRegistry xContentRegistry, + ActionFilters actionFilters + ) { + super(ListIOCsAction.NAME, transportService, actionFilters, ListIOCsActionRequest::new); + this.client = client; + this.xContentRegistry = xContentRegistry; + this.threadPool = this.client.threadPool(); + } + + @Override + protected void doExecute(Task task, ListIOCsActionRequest request, ActionListener listener) { + AsyncListIOCsAction asyncAction = new AsyncListIOCsAction(task, request, listener); + asyncAction.start(); + } + + class AsyncListIOCsAction { + private ListIOCsActionRequest request; + private ActionListener listener; + + private final AtomicReference response; + private final AtomicBoolean counter = new AtomicBoolean(); + private final Task task; + + AsyncListIOCsAction(Task task, ListIOCsActionRequest request, ActionListener listener) { + this.task = task; + this.request = request; + this.listener = listener; + this.response = new AtomicReference<>(); + } + + void start() { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + if (!ListIOCsActionRequest.ALL_TYPES_FILTER.equalsIgnoreCase(request.getType())) { + boolQueryBuilder.filter(QueryBuilders.termQuery(STIX2_IOC_NESTED_PATH + STIX2IOC.TYPE_FIELD, request.getType())); + } + + if (request.getFeedId() != null && !request.getFeedId().isBlank()) { + boolQueryBuilder.filter(QueryBuilders.termQuery(STIX2_IOC_NESTED_PATH + STIX2IOC.FEED_ID_FIELD, request.getFeedId())); + } + + if (!request.getSearch().isEmpty()) { + boolQueryBuilder.must( + QueryBuilders.queryStringQuery(request.getSearch()) + .defaultOperator(Operator.OR) +// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.ID_FIELD) // Currently not a column in UX table + .field(STIX2_IOC_NESTED_PATH + STIX2IOC.NAME_FIELD) + .field(STIX2_IOC_NESTED_PATH + STIX2IOC.TYPE_FIELD) + .field(STIX2_IOC_NESTED_PATH + STIX2IOC.VALUE_FIELD) + .field(STIX2_IOC_NESTED_PATH + STIX2IOC.SEVERITY_FIELD) + .field(STIX2_IOC_NESTED_PATH + STIX2IOC.CREATED_FIELD) + .field(STIX2_IOC_NESTED_PATH + STIX2IOC.MODIFIED_FIELD) +// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.DESCRIPTION_FIELD) // Currently not a column in UX table +// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.LABELS_FIELD) // Currently not a column in UX table +// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.SPEC_VERSION_FIELD) // Currently not a column in UX table + ); + } + + + + SortBuilder sortBuilder = SortBuilders + .fieldSort(STIX2_IOC_NESTED_PATH + request.getSortString()) + .order(SortOrder.fromString(request.getSortOrder().toString())); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .version(true) + .seqNoAndPrimaryTerm(true) + .fetchSource(true) + .query(boolQueryBuilder) + .sort(sortBuilder) + .size(request.getSize()) + .from(request.getStartIndex()); + + SearchRequest searchRequest = new SearchRequest() + .indices(STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN) + .source(searchSourceBuilder) + .preference(Preference.PRIMARY_FIRST.type()); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + if (searchResponse.isTimedOut()) { + onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT)); + } + List iocs = new ArrayList<>(); + Arrays.stream(searchResponse.getHits().getHits()) + .forEach(hit -> { + try { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceAsString()); + xcp.nextToken(); + + STIX2IOCDto ioc = STIX2IOCDto.parse(xcp, hit.getId(), hit.getVersion()); + iocs.add(ioc); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage( + "Failed to parse IOC doc from hit {}", hit), + e + ); + } + }); + onOperation(new ListIOCsActionResponse(searchResponse.getHits().getTotalHits().value, iocs)); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + // If no IOC system indexes are found, return empty list response + listener.onResponse(ListIOCsActionResponse.EMPTY_RESPONSE); + } else { + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + } + }); + } + + private void onOperation(ListIOCsActionResponse response) { + this.response.set(response); + if (counter.compareAndSet(false, true)) { + finishHim(response, null); + } + } + + private void onFailures(Exception t) { + if (counter.compareAndSet(false, true)) { + finishHim(null, t); + } + } + + private void finishHim(ListIOCsActionResponse response, Exception t) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> { + if (t != null) { + if (t instanceof OpenSearchStatusException) { + throw t; + } + throw SecurityAnalyticsException.wrap(t); + } else { + return response; + } + })); + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java new file mode 100644 index 000000000..a99e04090 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.junit.After; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.client.WarningFailureException; +import org.opensearch.common.settings.Settings; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.services.STIX2IOCFeedStore; +import org.opensearch.securityanalytics.util.STIX2IOCGenerator; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class ListIOCsRestApiIT extends SecurityAnalyticsRestTestCase { + private final String indexMapping = "\"properties\": {\n" + + " \"stix2_ioc\": {\n" + + " \"dynamic\": \"false\",\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"value\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"severity\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"spec_version\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"created\": {\n" + + " \"type\": \"date\",\n" + + " \"format\": \"strict_date_time||epoch_millis\"\n" + + " },\n" + + " \"modified\": {\n" + + " \"type\": \"date\",\n" + + " \"format\": \"strict_date_optional_time||epoch_millis\"\n" + + " },\n" + + " \"description\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"labels\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"feed_id\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + + private String testFeedSourceConfigId; + private String indexName; + ListIOCsActionRequest request; + + @After + public void cleanUp() throws IOException { + deleteIndex(indexName); + + testFeedSourceConfigId = null; + indexName = null; + request = null; + } + + public void test_retrievesIOCs() throws IOException { + // Create index with mappings + testFeedSourceConfigId = TestHelpers.randomLowerCaseString(); + indexName = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); + + try { + createIndex(indexName, Settings.EMPTY, indexMapping); + } catch (WarningFailureException warningFailureException) { + // Warns that index names starting with "." will be deprecated, but still creates the index + } catch (Exception e) { + fail(String.format("Test index creation failed with error: %s", e)); + } + + // Ingest IOCs + List iocs = IntStream.range(0, randomInt(5)) + .mapToObj(i -> STIX2IOCGenerator.randomIOC()) + .collect(Collectors.toList()); + for (STIX2IOC ioc : iocs) { + indexDoc(indexName, "", STIX2IOCGenerator.toJsonString(ioc)); + } + + request = new ListIOCsActionRequest( + 0, + iocs.size() + 1, + ListIOCsActionRequest.SortOrder.asc.toString(), + STIX2.NAME_FIELD, + "", + "ALL", + "" + ); + + // Retrieve IOCs + Response response = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(request), Collections.emptyMap(), null); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + Map respMap = asMap(response); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> hits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), hits.size()); + + // Sort for easy comparison + iocs.sort(Comparator.comparing(STIX2IOC::getName)); + hits.sort(Comparator.comparing(hit -> (String) hit.get(STIX2IOC.NAME_FIELD))); + + for (int i = 0; i < iocs.size(); i++) { + Map hit = hits.get(i); + STIX2IOC newIoc = new STIX2IOC( + (String) hit.get(STIX2IOC.ID_FIELD), + (String) hit.get(STIX2IOC.NAME_FIELD), + IOCType.valueOf((String) hit.get(STIX2IOC.TYPE_FIELD)), + (String) hit.get(STIX2IOC.VALUE_FIELD), + (String) hit.get(STIX2IOC.SEVERITY_FIELD), + Instant.parse((String) hit.get(STIX2IOC.CREATED_FIELD)), + Instant.parse((String) hit.get(STIX2IOC.MODIFIED_FIELD)), + (String) hit.get(STIX2IOC.DESCRIPTION_FIELD), + (List) hit.get(STIX2IOC.LABELS_FIELD), + (String) hit.get(STIX2IOC.FEED_ID_FIELD), + (String) hit.get(STIX2IOC.SPEC_VERSION_FIELD), + Long.parseLong(String.valueOf(hit.get(STIX2IOC.VERSION_FIELD))) + ); + STIX2IOCGenerator.assertEqualIOCs(iocs.get(i), newIoc); + } + } + + // TODO: Implement additional tests using various query param combinations +} diff --git a/src/test/java/org/opensearch/securityanalytics/services/STIX2IOCFetchServiceIT.java b/src/test/java/org/opensearch/securityanalytics/services/STIX2IOCFetchServiceIT.java deleted file mode 100644 index d2b41f21b..000000000 --- a/src/test/java/org/opensearch/securityanalytics/services/STIX2IOCFetchServiceIT.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.services; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.opensearch.action.delete.DeleteRequest; -import org.opensearch.core.action.ActionListener; -import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; -import org.opensearch.securityanalytics.commons.utils.testUtils.S3ObjectGenerator; -import org.opensearch.securityanalytics.model.STIX2IOC; -import org.opensearch.securityanalytics.model.STIX2IOCDto; -import org.opensearch.securityanalytics.util.STIX2IOCGenerator; -import org.opensearch.test.OpenSearchIntegTestCase; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; - -import java.io.IOException; -import java.util.Locale; -import java.util.UUID; - -public class STIX2IOCFetchServiceIT extends OpenSearchIntegTestCase { - private String bucket; - private String region; - private String roleArn; - - private S3Client s3Client; - private S3ObjectGenerator s3ObjectGenerator; - private STIX2IOCFetchService service; - - private String testFeedSourceConfigId; - private String testIndex; - private S3ConnectorConfig s3ConnectorConfig; - - @Before - public void beforeTest() { - if (service == null) { - region = System.getProperty("tests.STIX2IOCFetchServiceIT.region"); - roleArn = System.getProperty("tests.STIX2IOCFetchServiceIT.roleArn"); - bucket = System.getProperty("tests.STIX2IOCFetchServiceIT.bucket"); - - s3Client = S3Client.builder() - .region(Region.of(region)) - .build(); - s3ObjectGenerator = new S3ObjectGenerator(s3Client, bucket); - - service = new STIX2IOCFetchService(); - } - testFeedSourceConfigId = UUID.randomUUID().toString(); - testIndex = null; - s3ConnectorConfig = new S3ConnectorConfig(bucket, testFeedSourceConfigId, region, roleArn); - } - - @After - private void afterTest() { - if (testIndex != null && !testIndex.isBlank()) { - client().delete(new DeleteRequest(testIndex)); - } - } - - @Test - public void test_fetchIocs_fetchesIocsCorrectly() throws IOException { - int numOfIOCs = 5; - s3ObjectGenerator.write(numOfIOCs, testFeedSourceConfigId, new STIX2IOCGenerator()); - - ActionListener listener = new ActionListener<>() { - @Override - public void onResponse(STIX2IOCFetchService.STIX2IOCFetchResponse stix2IOCFetchResponse) { - assertEquals(numOfIOCs, stix2IOCFetchResponse.getIocs().size()); - //TODO hurneyt need to retrieve the test IOCs from s3ObjectGenerator.write, and compare to output - } - - @Override - public void onFailure(Exception e) { - fail("STIX2IOCFetchService.fetchIocs failed with error: " + e); - } - }; - - service.fetchIocs(s3ConnectorConfig, listener); - } - - - // TODO hurneyt extract feedIndexExists and initFeedIndex to helper function, or expose for testing -// @Test -// public void test_hasIocSystemIndex_returnsFalse_whenIndexNotCreated() throws ExecutionException, InterruptedException { -// // Confirm index doesn't exist before running test case -// testIndex = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); -// ClusterHealthResponse clusterHealthResponse = client().admin().cluster().health(new ClusterHealthRequest()).get(); -// assertFalse(clusterHealthResponse.getIndices().containsKey(testIndex)); -// -// // Run test case -// assertFalse(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_hasIocSystemIndex_returnsFalse_withInvalidIndex() throws ExecutionException, InterruptedException { -// // Create test index -// testIndex = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); -// client().admin().indices().create(new CreateIndexRequest(testIndex)).get(); -// -// // Run test case -// assertFalse(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_hasIocSystemIndex_returnsTrue_whenIndexExists() throws ExecutionException, InterruptedException { -// // Create test index -// testIndex = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); -// client().admin().indices().create(new CreateIndexRequest(testIndex)).get(); -// -// // Run test case -// assertTrue(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_initSystemIndexes_createsIndexes() { -// // Confirm index doesn't exist -// testIndex = IocService.getFeedConfigIndexName(testFeedSourceConfigId); -// assertFalse(service.feedIndexExists(testIndex)); -// -// // Run test case -// service.initFeedIndex(testIndex, new ActionListener<>() { -// @Override -// public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) {} -// -// @Override -// public void onFailure(Exception e) { -// fail(String.format("Creation of %s should not fail: %s", testIndex, e)); -// } -// }); -// assertTrue(service.feedIndexExists(testIndex)); -// } -// -// @Test -// public void test_indexIocs_ingestsIocsCorrectly() throws IOException { -// // Prepare test IOCs -// List iocs = IntStream.range(0, randomInt()) -// .mapToObj(i -> STIX2IOCGenerator.randomIOC()) -// .collect(Collectors.toList()); -// -// // Run test case -// service.indexIocs(testFeedSourceConfigId, iocs, new ActionListener<>() { -// @Override -// public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) { -// // Confirm expected number of IOCs in response -// assertEquals(iocs.size(), fetchIocsActionResponse.getIocs().size()); -// -// try { -// // Search system indexes directly -// SearchRequest searchRequest = new SearchRequest() -// .indices(IOC_ALL_INDEX_PATTERN) -// .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())); -// SearchResponse searchResponse = client().search(searchRequest).get(); -// -// // Confirm expected number of hits -// assertEquals(iocs.size(), searchResponse.getHits().getHits().length); -// -// // Parse hits to IOCs -// List iocHits = Collections.emptyList(); -// for (SearchHit ioc : searchResponse.getHits()) { -// try { -// iocHits.add(IocModel.parse(TestHelpers.parser(ioc.getSourceAsString()), null)); -// } catch (IOException e) { -// fail(String.format("Failed to parse IOC hit: %s", e)); -// } -// } -// -// // Confirm expected number of IOCs -// assertEquals(iocs.size(), iocHits.size()); -// -// // Sort IOCs for comparison -// iocs.sort(Comparator.comparing(IocModel::getId)); -// fetchIocsActionResponse.getIocs().sort(Comparator.comparing(IocDto::getId)); -// iocHits.sort(Comparator.comparing(IocModel::getId)); -// -// // Confirm IOCs are equal -// for (int i = 0; i < iocs.size(); i++) { -// assertEqualIocs(iocs.get(i), fetchIocsActionResponse.getIocs().get(i)); -// IocModelTests.assertEqualIOCs(iocs.get(i), iocHits.get(i)); -// } -// } catch (InterruptedException | ExecutionException e) { -// fail(String.format("IOC_ALL_INDEX_PATTERN search failed: %s", e)); -// } -// } -// -// @Override -// public void onFailure(Exception e) { -// fail(String.format("Ingestion of IOCs should not fail: %s", e)); -// } -// }); -// } - - private String createEndpointString() { - return STIX2IOCServiceTestAPI.RestSTIX2IOCServiceTestAPIAction.ROUTE + String.format(Locale.getDefault(), - "?%s=%s&%s=%s&%s=%s&%s=%s", - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.BUCKET_FIELD, - bucket, - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.REGION_FIELD, - region, - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.ROLE_ARN_FIELD, - roleArn, - STIX2IOCServiceTestAPI.STIX2IOCServiceTestAPIRequest.OBJECT_KEY_FIELD, - testFeedSourceConfigId - ); - } -} diff --git a/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java index 267c2b324..a1fcfae69 100644 --- a/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java +++ b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java @@ -10,6 +10,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.ListIOCsActionRequest; import org.opensearch.securityanalytics.commons.model.IOC; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.utils.testUtils.PojoGenerator; @@ -25,6 +27,7 @@ import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.opensearch.securityanalytics.TestHelpers.randomLowerCaseString; import static org.opensearch.test.OpenSearchTestCase.randomInt; import static org.opensearch.test.OpenSearchTestCase.randomLong; @@ -222,7 +225,7 @@ public static void assertIOCEqualsDTO(STIX2IOC ioc, STIX2IOCDto iocDto) { } public static void assertEqualIOCs(STIX2IOC ioc, STIX2IOC newIoc) { - assertEquals(ioc.getId(), newIoc.getId()); + assertNotNull(newIoc.getId()); assertEquals(ioc.getName(), newIoc.getName()); assertEquals(ioc.getValue(), newIoc.getValue()); assertEquals(ioc.getSeverity(), newIoc.getSeverity()); @@ -235,7 +238,7 @@ public static void assertEqualIOCs(STIX2IOC ioc, STIX2IOC newIoc) { } public static void assertEqualIocDtos(STIX2IOCDto ioc, STIX2IOCDto newIoc) { - assertEquals(ioc.getId(), newIoc.getId()); + assertNotNull(newIoc.getId()); assertEquals(ioc.getName(), newIoc.getName()); assertEquals(ioc.getValue(), newIoc.getValue()); assertEquals(ioc.getSeverity(), newIoc.getSeverity()); @@ -246,4 +249,18 @@ public static void assertEqualIocDtos(STIX2IOCDto ioc, STIX2IOCDto newIoc) { assertEquals(ioc.getFeedId(), newIoc.getFeedId()); assertEquals(ioc.getSpecVersion(), newIoc.getSpecVersion()); } + + public static String getListIOCsURI(ListIOCsActionRequest request) { + return String.format( + "%s?%s=%s&%s=%s&%s=%s&%s=%s&%s=%s&%s=%s&%s=%s", + SecurityAnalyticsPlugin.LIST_IOCS_URI, + ListIOCsActionRequest.START_INDEX_FIELD, request.getStartIndex(), + ListIOCsActionRequest.SIZE_FIELD, request.getSize(), + ListIOCsActionRequest.SORT_ORDER_FIELD, request.getSortOrder(), + ListIOCsActionRequest.SORT_STRING_FIELD, request.getSortString(), + ListIOCsActionRequest.SEARCH_FIELD, request.getSearch(), + ListIOCsActionRequest.TYPE_FIELD, request.getType(), + STIX2IOC.FEED_ID_FIELD, request.getFeedId() + ); + } }