diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index ef1c093d9..a49c82852 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -103,7 +103,6 @@ import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; @@ -117,7 +116,6 @@ import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatch.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java similarity index 92% rename from src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatch.java rename to src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java index 037541741..43ecd53ad 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatch.java +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java @@ -20,7 +20,7 @@ * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job * It's the inverse of an IoC finding which maps a document to list of IoC's */ -public class IocMatch implements Writeable, ToXContent { +public class IocFinding implements Writeable, ToXContent { //TODO implement IoC_Match interface from security-analytics-commons public static final String ID_FIELD = "id"; public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; @@ -42,8 +42,8 @@ public class IocMatch implements Writeable, ToXContent { private final Instant timestamp; private final String executionId; - public IocMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, - String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { + public IocFinding(String id, List relatedDocIds, List feedIds, String iocScanJobId, + String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); this.id = id; this.relatedDocIds = relatedDocIds; @@ -56,7 +56,7 @@ public IocMatch(String id, List relatedDocIds, List feedIds, Str this.executionId = executionId; } - public IocMatch(StreamInput in) throws IOException { + public IocFinding(StreamInput in) throws IOException { id = in.readString(); relatedDocIds = in.readStringList(); feedIds = in.readStringList(); @@ -133,7 +133,7 @@ public String getExecutionId() { return executionId; } - public static IocMatch parse(XContentParser xcp) throws IOException { + public static IocFinding parse(XContentParser xcp) throws IOException { String id = null; List relatedDocIds = new ArrayList<>(); List feedIds = new ArrayList<>(); @@ -197,11 +197,11 @@ public static IocMatch parse(XContentParser xcp) throws IOException { } } - return new IocMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + return new IocFinding(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); } - public static IocMatch readFrom(StreamInput in) throws IOException { - return new IocMatch(in); + public static IocFinding readFrom(StreamInput in) throws IOException { + return new IocFinding(in); } diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatchWithDocs.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatchWithDocs.java deleted file mode 100644 index 30d374be5..000000000 --- a/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatchWithDocs.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.securityanalytics.model.threatintel; - -import org.opensearch.commons.alerting.model.FindingDocument; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.xcontent.XContentParserUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class IocMatchWithDocs implements Writeable, ToXContent { - - private static final String IOC_MATCH_FIELD = "ioc_finding"; - - private static final String DOCUMENTS_FIELD = "document_list"; - - private IocMatch iocMatch; - - private List documents; - - public IocMatchWithDocs(IocMatch iocMatch, List documents) { - super(); - this.iocMatch = iocMatch; - this.documents = documents; - } - - public IocMatchWithDocs(StreamInput sin) throws IOException { - this( - IocMatch.readFrom(sin), - sin.readList(FindingDocument::readFrom) - ); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - iocMatch.writeTo(out); - out.writeCollection(documents); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject() - .field(IOC_MATCH_FIELD, iocMatch) - .field(DOCUMENTS_FIELD, documents); - return builder.endObject(); - } - - public static IocMatchWithDocs parse(XContentParser xcp) throws IOException { - IocMatch iocMatch = null; - List documents = new ArrayList<>(); - - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); - while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { - String fieldName = xcp.currentName(); - xcp.nextToken(); - - switch (fieldName) { - case IOC_MATCH_FIELD: - iocMatch = IocMatch.parse(xcp); - break; - case DOCUMENTS_FIELD: - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); - while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - documents.add(FindingDocument.parse(xcp)); - } - break; - default: - xcp.skipChildren(); - } - } - return new IocMatchWithDocs(iocMatch, documents); - } - - public static IocMatchWithDocs readFrom(StreamInput sin) throws IOException { - return new IocMatchWithDocs(sin); - } -} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java index 880acca1d..a33b69401 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java @@ -24,23 +24,13 @@ public class GetIocFindingsRequest extends ActionRequest { private Instant endTime; - private String threatIntelMonitorId; - private Table table; - public static final String THREAT_INTEL_MONITOR_ID = "monitor_id"; - - public GetIocFindingsRequest(String threatIntelMonitorId) { - super(); - this.threatIntelMonitorId = threatIntelMonitorId; - } - public GetIocFindingsRequest(StreamInput sin) throws IOException { this( sin.readOptionalStringList(), sin.readOptionalInstant(), sin.readOptionalInstant(), - sin.readOptionalString(), Table.readFrom(sin) ); } @@ -48,22 +38,17 @@ public GetIocFindingsRequest(StreamInput sin) throws IOException { public GetIocFindingsRequest(List findingIds, Instant startTime, Instant endTime, - String threatIntelMonitorId, Table table) { this.findingIds = findingIds; this.startTime = startTime; this.endTime = endTime; - this.threatIntelMonitorId = threatIntelMonitorId; this.table = table; } @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (threatIntelMonitorId != null && threatIntelMonitorId.isEmpty()) { - validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(), - "threat intel monitor id is missing"), validationException); - } else if (startTime != null && endTime != null && startTime.isAfter(endTime)) { + if (startTime != null && endTime != null && startTime.isAfter(endTime)) { validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(), "startTime should be less than endTime"), validationException); } @@ -75,7 +60,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalStringCollection(findingIds); out.writeOptionalInstant(startTime); out.writeOptionalInstant(endTime); - out.writeOptionalString(threatIntelMonitorId); table.writeTo(out); } @@ -91,10 +75,6 @@ public Instant getEndTime() { return endTime; } - public String getThreatIntelMonitorId() { - return threatIntelMonitorId; - } - public Table getTable() { return table; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java index b34bf76bb..50ae08dd4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java @@ -9,8 +9,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.securityanalytics.model.threatintel.IocMatch; -import org.opensearch.securityanalytics.model.threatintel.IocMatchWithDocs; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import java.io.IOException; import java.util.Collections; @@ -24,9 +23,9 @@ public class GetIocFindingsResponse extends ActionResponse implements ToXContent private Integer totalFindings; - private List iocFindings; + private List iocFindings; - public GetIocFindingsResponse(Integer totalFindings, List iocFindings) { + public GetIocFindingsResponse(Integer totalFindings, List iocFindings) { super(); this.totalFindings = totalFindings; this.iocFindings = iocFindings; @@ -35,7 +34,7 @@ public GetIocFindingsResponse(Integer totalFindings, List iocF public GetIocFindingsResponse(StreamInput sin) throws IOException { this( sin.readInt(), - Collections.unmodifiableList(sin.readList(IocMatchWithDocs::new)) + Collections.unmodifiableList(sin.readList(IocFinding::new)) ); } @@ -57,7 +56,7 @@ public Integer getTotalFindings() { return totalFindings; } - public List getIocFindings() { + public List getIocFindings() { return iocFindings; } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java similarity index 83% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java index 1e87f1927..94f2ef540 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java @@ -24,16 +24,14 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.model.threatintel.IocMatch; -import org.opensearch.securityanalytics.model.threatintel.IocMatchWithDocs; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; -import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import org.opensearch.threadpool.ThreadPool; import java.io.BufferedReader; import java.io.IOException; @@ -47,22 +45,22 @@ /** * Data layer to perform CRUD operations for threat intel ioc match : store in system index. */ -public class IocMatchService { +public class IocFindingService { //TODO manage index rollover public static final String INDEX_NAME = ".opensearch-sap-iocmatch"; - private static final Logger log = LogManager.getLogger(IocMatchService.class); + private static final Logger log = LogManager.getLogger(IocFindingService.class); private final Client client; private final ClusterService clusterService; private final NamedXContentRegistry xContentRegistry; - public IocMatchService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { + public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { this.client = client; this.clusterService = clusterService; this.xContentRegistry = xContentRegistry; } - public void indexIocMatches(List iocMatches, + public void indexIocMatches(List iocFindings, final ActionListener actionListener) { try { Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); @@ -70,16 +68,16 @@ public void indexIocMatches(List iocMatches, r -> { List bulkRequestList = new ArrayList<>(); BulkRequest bulkRequest = new BulkRequest(INDEX_NAME); - for (int i = 0; i < iocMatches.size(); i++) { - IocMatch iocMatch = iocMatches.get(i); + for (int i = 0; i < iocFindings.size(); i++) { + IocFinding iocFinding = iocFindings.get(i); try { IndexRequest indexRequest = new IndexRequest(INDEX_NAME) - .source(iocMatch.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) .opType(DocWriteRequest.OpType.CREATE); bulkRequest.add(indexRequest); if ( bulkRequest.requests().size() == batchSize - && i != iocMatches.size() - 1 // final bulk request will be added outside for loop with refresh policy none + && i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none ) { bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); bulkRequestList.add(bulkRequest); @@ -122,7 +120,7 @@ public void indexIocMatches(List iocMatches, private String getIndexMapping() { try { - try (InputStream is = IocMatchService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { return reader.lines().map(String::trim).collect(Collectors.joining()); } @@ -178,24 +176,16 @@ public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final Acti public void onResponse(SearchResponse searchResponse) { try { long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; - MultiGetRequest mGetRequest = new MultiGetRequest(); - List iocMatchWithDocs = new ArrayList<>(); - List iocMatches = new ArrayList<>(); + List iocFindings = new ArrayList<>(); for (SearchHit hit: searchResponse.getHits()) { XContentParser xcp = XContentType.JSON.xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); - IocMatch iocMatch = IocMatch.parse(xcp); - iocMatches.add(iocMatch); - - List documentIds = iocMatch.getRelatedDocIds(); - } - - for (IocMatch iocMatch: iocMatches) { - iocMatchWithDocs.add(new IocMatchWithDocs(iocMatch, List.of())); + IocFinding iocFinding = IocFinding.parse(xcp); + iocFindings.add(iocFinding); } - actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocMatchWithDocs)); + actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); } catch (Exception ex) { this.onFailure(ex); } @@ -203,6 +193,10 @@ public void onResponse(SearchResponse searchResponse) { @Override public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); + return; + } actionListener.onFailure(e); } }); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java index eb8c14552..294e03e65 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -4,7 +4,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.commons.alerting.model.Finding; -import org.opensearch.securityanalytics.model.threatintel.IocMatch; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeFieldMappings; import org.opensearch.securityanalytics.threatIntel.iocscan.model.Ioc; @@ -115,7 +115,7 @@ public void createIoCMatches(List iocs, Map> iocValueTo .add(ioc.getFeedId()); } - List iocMatches = new ArrayList<>(); + List iocFindings = new ArrayList<>(); for (Map.Entry> entry : iocValueToFeedIds.entrySet()) { String iocValue = entry.getKey(); @@ -124,7 +124,7 @@ public void createIoCMatches(List iocs, Map> iocValueTo List relatedDocIds = new ArrayList<>(iocValueToDocIdMap.getOrDefault(iocValue, new HashSet<>())); List feedIdsList = new ArrayList<>(feedIds); try { - IocMatch iocMatch = new IocMatch( + IocFinding iocFinding = new IocFinding( UUID.randomUUID().toString(), // Generating a unique ID relatedDocIds, feedIdsList, @@ -135,7 +135,7 @@ public void createIoCMatches(List iocs, Map> iocValueTo timestamp, UUID.randomUUID().toString() // TODO execution ID ); - iocMatches.add(iocMatch); + iocFindings.add(iocFinding); } catch (Exception e) { log.error(String.format("skipping creating ioc match for %s due to unexpected failure.", entry.getKey()), e); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java index 0746eddf4..1826824d3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java @@ -1,10 +1,7 @@ package org.opensearch.securityanalytics.threatIntel.iocscan.service; -import org.opensearch.commons.alerting.model.Finding; -import org.opensearch.securityanalytics.model.threatintel.IocMatch; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; -import java.util.List; import java.util.function.BiConsumer; public interface IoCScanServiceInterface { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java index 879cb10c1..a6077679c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java @@ -11,6 +11,7 @@ import org.opensearch.rest.action.RestToXContentListener; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; import java.io.IOException; @@ -31,8 +32,6 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - String threatIntelMonitorId = request.param("monitor_id"); - String sortString = request.param("sortString", "timestamp"); String sortOrder = request.param("sortOrder", "asc"); String missing = request.param("missing"); @@ -81,11 +80,10 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli findingIds, startTime, endTime, - threatIntelMonitorId, table ); return channel -> client.execute( - GetFindingsAction.INSTANCE, + GetIocFindingsAction.INSTANCE, getIocFindingsRequest, new RestToXContentListener<>(channel) ); @@ -93,7 +91,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @Override public List routes() { - return singletonList(new Route(GET, SecurityAnalyticsPlugin.IOC_BASE_URI + "/findings" + "/_search")); + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings" + "/_search")); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java index aae421d9b..debde4655 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -31,7 +31,7 @@ import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; -import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocMatchService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -42,7 +42,7 @@ public class TransportGetIocFindingsAction extends HandledTransportAction implements SecureTransportAction { - private final IocMatchService iocMatchService; + private final IocFindingService iocFindingService; private final ClusterService clusterService; @@ -65,7 +65,7 @@ public TransportGetIocFindingsAction( this.settings = settings; this.clusterService = clusterService; this.threadPool = client.threadPool(); - this.iocMatchService = new IocMatchService(client, this.clusterService, xContentRegistry); + this.iocFindingService = new IocFindingService(client, this.clusterService, xContentRegistry); this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); } @@ -134,7 +134,7 @@ protected void doExecute(Task task, GetIocFindingsRequest request, ActionListene searchSourceBuilder.query(queryBuilder).trackTotalHits(true); this.threadPool.getThreadContext().stashContext(); - iocMatchService.searchIocMatches(searchSourceBuilder, actionListener); + iocFindingService.searchIocMatches(searchSourceBuilder, actionListener); } private void setFilterByEnabled(boolean filterByEnabled) { diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 91289a91e..d06237e57 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -67,6 +67,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -667,6 +668,10 @@ protected HttpEntity toHttpEntity(SATIFSourceConfigDto SaTifSourceConfigDto) thr return new StringEntity(toJsonString(SaTifSourceConfigDto), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { + return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); + } + protected RestStatus restStatus(Response response) { return RestStatus.fromCode(response.getStatusLine().getStatusCode()); } @@ -715,6 +720,11 @@ private String toJsonString(SATIFSourceConfigDto SaTifSourceConfigDto) throws IO return IndexUtilsKt.string(shuffleXContent(SaTifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String alertingScheduledJobMappings() { return " \"_meta\" : {\n" + " \"schema_version\": 5\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index f5bcd77bf..ba2a15d63 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -31,7 +31,7 @@ import org.opensearch.securityanalytics.model.IocDao; import org.opensearch.securityanalytics.model.IocDto; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.model.threatintel.IocMatch; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; @@ -807,9 +807,9 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } - public static String toJsonString(IocMatch iocMatch) throws IOException { + public static String toJsonString(IocFinding iocFinding) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); - builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder = iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS); return BytesReference.bytes(builder).utf8ToString(); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocMatchTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java similarity index 50% rename from src/test/java/org/opensearch/securityanalytics/model/IocMatchTests.java rename to src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java index 298d3f7e4..cbad9f5d2 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/IocMatchTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java @@ -5,7 +5,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.securityanalytics.model.threatintel.IocMatch; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -14,24 +14,24 @@ import static org.opensearch.securityanalytics.TestHelpers.toJsonString; -public class IocMatchTests extends OpenSearchTestCase { +public class IocFindingTests extends OpenSearchTestCase { public void testIoCMatchAsAStream() throws IOException { - IocMatch iocMatch = getRandomIoCMatch(); - String jsonString = toJsonString(iocMatch); + IocFinding iocFinding = getRandomIoCMatch(); + String jsonString = toJsonString(iocFinding); BytesStreamOutput out = new BytesStreamOutput(); - iocMatch.writeTo(out); + iocFinding.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IocMatch newIocMatch = new IocMatch(sin); - assertEquals(iocMatch.getId(), newIocMatch.getId()); - assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); - assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); - assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); - assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); - assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); - assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); - assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); - assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getIocScanJobId(), newIocFinding.getIocScanJobId()); + assertEquals(iocFinding.getIocScanJobName(), newIocFinding.getIocScanJobName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); } public void testIoCMatchParse() throws IOException { @@ -39,20 +39,20 @@ public void testIoCMatchParse() throws IOException { "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; - IocMatch iocMatch = IocMatch.parse((getParser(iocMatchString))); + IocFinding iocFinding = IocFinding.parse((getParser(iocMatchString))); BytesStreamOutput out = new BytesStreamOutput(); - iocMatch.writeTo(out); + iocFinding.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IocMatch newIocMatch = new IocMatch(sin); - assertEquals(iocMatch.getId(), newIocMatch.getId()); - assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); - assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); - assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); - assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); - assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); - assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); - assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); - assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getIocScanJobId(), newIocFinding.getIocScanJobId()); + assertEquals(iocFinding.getIocScanJobName(), newIocFinding.getIocScanJobName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); } public XContentParser getParser(String xc) throws IOException { @@ -62,8 +62,8 @@ public XContentParser getParser(String xc) throws IOException { } - private static IocMatch getRandomIoCMatch() { - return new IocMatch( + private static IocFinding getRandomIoCMatch() { + return new IocFinding( randomAlphaOfLength(10), List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchServiceIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceIT.java similarity index 70% rename from src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchServiceIT.java rename to src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceIT.java index 4c7007aef..7f3dfcba5 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchServiceIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceIT.java @@ -1,33 +1,30 @@ package org.opensearch.securityanalytics.threatIntel.iocscan.dao; -import org.opensearch.action.LatchedActionListener; -import org.opensearch.action.StepListener; import org.opensearch.action.search.SearchRequest; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.action.ActionListener; import org.opensearch.securityanalytics.SecurityAnalyticsIntegTestCase; -import org.opensearch.securityanalytics.model.threatintel.IocMatch; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CountDownLatch; -public class IocMatchServiceIT extends SecurityAnalyticsIntegTestCase { +public class IocFindingServiceIT extends SecurityAnalyticsIntegTestCase { public void test_indexIocMatches() throws InterruptedException { - IocMatchService service = new IocMatchService(client(), clusterService(), xContentRegistry()); - List iocMatches = generateIocMatches(10); + IocFindingService service = new IocFindingService(client(), clusterService(), xContentRegistry()); + List iocFindings = generateIocMatches(10); CountDown countdown = new CountDown(1); - service.indexIocMatches(iocMatches, ActionListener.wrap(r -> { + service.indexIocMatches(iocFindings, ActionListener.wrap(r -> { countdown.countDown(); }, e -> { logger.error("failed to index ioc matches", e); fail(); countdown.countDown(); })); - SearchRequest request = new SearchRequest(IocMatchService.INDEX_NAME); + SearchRequest request = new SearchRequest(IocFindingService.INDEX_NAME); request.source().size(10); CountDown countDownLatch1 = new CountDown(1); client().search(request, ActionListener.wrap( @@ -45,12 +42,12 @@ public void test_indexIocMatches() throws InterruptedException { countDownLatch1.isCountedDown(); } - private List generateIocMatches(int i) { - List iocMatches = new ArrayList<>(); + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); String monitorId = randomAlphaOfLength(10); String monitorName = randomAlphaOfLength(10); for (int i1 = 0; i1 < i; i1++) { - iocMatches.add(new IocMatch( + iocFindings.add(new IocFinding( randomAlphaOfLength(10), randomList(1, 10, () -> randomAlphaOfLength(10)),//docids randomList(1, 10, () -> randomAlphaOfLength(10)), //feedids @@ -62,6 +59,6 @@ private List generateIocMatches(int i) { randomAlphaOfLength(10) )); } - return iocMatches; + return iocFindings; } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java new file mode 100644 index 000000000..22bd9536f --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class IocFindingServiceRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testGetIocFindings() throws IOException { + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.INDEX_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(5, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> randomAlphaOfLength(10)), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } +} \ No newline at end of file