diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index efe4ba836..cf1ce83a1 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -7,7 +7,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import java.util.Optional; @@ -62,7 +61,6 @@ import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.model.CustomLogType; -import org.opensearch.securityanalytics.model.IocDao; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.*; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; @@ -111,15 +109,6 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; public static final Map TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true); - public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-iocs"; - public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; - public static final String IOC_DOMAIN_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.DOMAIN.name().toLowerCase(Locale.ROOT); - public static final String IOC_HASH_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.HASH.name().toLowerCase(Locale.ROOT); - public static final String IOC_IP_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.IP.name().toLowerCase(Locale.ROOT); - - // TODO hurneyt IOC system index names should be refactored to include the timestamp instead of the IOC type - public static final String IOC_HISTORY_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_BASE + "-history-write"; - public static final String IOC_HISTORY_INDEX_PATTERN = "<." + IOC_INDEX_NAME_BASE + "-history-{now/d{yyyy.MM.dd.hh.mm.ss|UTC}}-1>"; private CorrelationRuleIndices correlationRuleIndices; diff --git a/src/main/java/org/opensearch/securityanalytics/action/FetchIocsActionResponse.java b/src/main/java/org/opensearch/securityanalytics/action/FetchIocsActionResponse.java new file mode 100644 index 000000000..8fbf3adb0 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/FetchIocsActionResponse.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.IOC; +import org.opensearch.securityanalytics.model.IocDto; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class FetchIocsActionResponse extends ActionResponse implements ToXContentObject { + public static String IOCS_FIELD = "iocs"; + public static String TOTAL_FIELD = "total"; + private List iocs = Collections.emptyList(); + + public FetchIocsActionResponse(List iocs) { + super(); + iocs.forEach( ioc -> this.iocs.add(new IocDto(ioc))); + } + + public FetchIocsActionResponse(StreamInput sin) throws IOException { + this(sin.readList(IOC::new)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(iocs); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IOCS_FIELD, this.iocs) + .field(TOTAL_FIELD, this.iocs.size()) + .endObject(); + } + + public List getIocs() { + return iocs; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocDao.java b/src/main/java/org/opensearch/securityanalytics/model/IOC.java similarity index 88% rename from src/main/java/org/opensearch/securityanalytics/model/IocDao.java rename to src/main/java/org/opensearch/securityanalytics/model/IOC.java index 4e00ec5d1..91cf8e3a6 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IocDao.java +++ b/src/main/java/org/opensearch/securityanalytics/model/IOC.java @@ -23,12 +23,8 @@ import java.util.Locale; import java.util.Objects; -import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_DOMAIN_INDEX_NAME; -import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_HASH_INDEX_NAME; -import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_IP_INDEX_NAME; - -public class IocDao implements Writeable, ToXContentObject { - private static final Logger logger = LogManager.getLogger(IocDao.class); +public class IOC implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(IOC.class); public static final String NO_ID = ""; @@ -56,7 +52,7 @@ public class IocDao implements Writeable, ToXContentObject { private List labels; private String feedId; - public IocDao( + public IOC( String id, String name, IocType type, @@ -83,7 +79,7 @@ public IocDao( validate(); } - public IocDao(StreamInput sin) throws IOException { + public IOC(StreamInput sin) throws IOException { this( sin.readString(), // id sin.readString(), // name @@ -99,7 +95,7 @@ public IocDao(StreamInput sin) throws IOException { ); } - public IocDao(IocDto iocDto) { + public IOC(IocDto iocDto) { this( iocDto.getId(), iocDto.getName(), @@ -115,8 +111,8 @@ public IocDao(IocDto iocDto) { ); } - public static IocDao readFrom(StreamInput sin) throws IOException { - return new IocDao(sin); + public static IOC readFrom(StreamInput sin) throws IOException { + return new IOC(sin); } @Override @@ -151,7 +147,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .endObject(); } - public static IocDao parse(XContentParser xcp, String id) throws IOException { + public static IOC parse(XContentParser xcp, String id) throws IOException { if (id == null) { id = NO_ID; } @@ -233,7 +229,7 @@ public static IocDao parse(XContentParser xcp, String id) throws IOException { } } - return new IocDao( + return new IOC( id, name, type, @@ -314,27 +310,10 @@ public String getFeedId() { } public enum IocType { - DOMAIN("domain") { - @Override - public String getSystemIndexName() { - return IOC_DOMAIN_INDEX_NAME; - } - }, - HASH("hash") { // TODO placeholder - @Override - public String getSystemIndexName() { - return IOC_HASH_INDEX_NAME; - } - }, - IP("ip") { - @Override - public String getSystemIndexName() { - return IOC_IP_INDEX_NAME; - } - }; + DOMAIN("domain"), + HASH("hash"), + IP("ip"); IocType(String type) {} - - public abstract String getSystemIndexName(); } } diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocDto.java b/src/main/java/org/opensearch/securityanalytics/model/IocDto.java index ca9163cf8..b104ebe9d 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IocDto.java +++ b/src/main/java/org/opensearch/securityanalytics/model/IocDto.java @@ -13,20 +13,17 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.xcontent.XContentParserUtils; import java.io.IOException; import java.time.Instant; -import java.util.Collections; import java.util.List; -import java.util.Locale; public class IocDto implements Writeable, ToXContentObject { private static final Logger logger = LogManager.getLogger(IocDto.class); private String id; private String name; - private IocDao.IocType type; + private IOC.IocType type; private String value; private String severity; private String specVersion; @@ -36,22 +33,22 @@ public class IocDto implements Writeable, ToXContentObject { private List labels; private String feedId; - public IocDto(IocDao iocDao) { - this.id = iocDao.getId(); - this.name = iocDao.getName(); - this.type = iocDao.getType(); - this.value = iocDao.getValue(); - this.severity = iocDao.getSeverity(); - this.specVersion = iocDao.getSpecVersion(); - this.created = iocDao.getCreated(); - this.modified = iocDao.getModified(); - this.description = iocDao.getDescription(); - this.labels = iocDao.getLabels(); - this.feedId = iocDao.getFeedId(); + public IocDto(IOC ioc) { + this.id = ioc.getId(); + this.name = ioc.getName(); + this.type = ioc.getType(); + this.value = ioc.getValue(); + this.severity = ioc.getSeverity(); + this.specVersion = ioc.getSpecVersion(); + this.created = ioc.getCreated(); + this.modified = ioc.getModified(); + this.description = ioc.getDescription(); + this.labels = ioc.getLabels(); + this.feedId = ioc.getFeedId(); } public IocDto(StreamInput sin) throws IOException { - this(new IocDao(sin)); + this(new IOC(sin)); } public static IocDto readFrom(StreamInput sin) throws IOException { @@ -76,22 +73,22 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject() - .field(IocDao.ID_FIELD, id) - .field(IocDao.NAME_FIELD, name) - .field(IocDao.TYPE_FIELD, type) - .field(IocDao.VALUE_FIELD, value) - .field(IocDao.SEVERITY_FIELD, severity) - .field(IocDao.SPEC_VERSION_FIELD, specVersion) - .timeField(IocDao.CREATED_FIELD, created) - .timeField(IocDao.MODIFIED_FIELD, modified) - .field(IocDao.DESCRIPTION_FIELD, description) - .field(IocDao.LABELS_FIELD, labels) - .field(IocDao.FEED_ID_FIELD, feedId) + .field(IOC.ID_FIELD, id) + .field(IOC.NAME_FIELD, name) + .field(IOC.TYPE_FIELD, type) + .field(IOC.VALUE_FIELD, value) + .field(IOC.SEVERITY_FIELD, severity) + .field(IOC.SPEC_VERSION_FIELD, specVersion) + .timeField(IOC.CREATED_FIELD, created) + .timeField(IOC.MODIFIED_FIELD, modified) + .field(IOC.DESCRIPTION_FIELD, description) + .field(IOC.LABELS_FIELD, labels) + .field(IOC.FEED_ID_FIELD, feedId) .endObject(); } public static IocDto parse(XContentParser xcp, String id) throws IOException { - return new IocDto(IocDao.parse(xcp, id)); + return new IocDto(IOC.parse(xcp, id)); } public String getId() { @@ -102,7 +99,7 @@ public String getName() { return name; } - public IocDao.IocType getType() { + public IOC.IocType getType() { return type; } diff --git a/src/main/java/org/opensearch/securityanalytics/services/IocService.java b/src/main/java/org/opensearch/securityanalytics/services/IocService.java index 371571848..542bb1e99 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/IocService.java +++ b/src/main/java/org/opensearch/securityanalytics/services/IocService.java @@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; @@ -21,26 +22,40 @@ import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.securityanalytics.action.FetchIocsActionResponse; -import org.opensearch.securityanalytics.model.IocDao; +import org.opensearch.securityanalytics.model.IOC; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; - -import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_INDEX_NAME_BASE; +import java.util.Locale; /** * IOC Service implements operations that interact with retrieving IOCs from data sources, - * parsing them into threat intel data models (i.e., [IocDao]), and ingesting them to system indexes. + * parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes. */ public class IocService { private final Logger log = LogManager.getLogger(IocService.class); + + public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-iocs"; + public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; + public static final String IOC_FEED_ID_PLACEHOLDER = "FEED_ID"; + public static final String IOC_INDEX_NAME_TEMPLATE = IOC_INDEX_NAME_BASE + "-" + IOC_FEED_ID_PLACEHOLDER; + + // TODO hurneyt implement history indexes + rollover logic + public static final String IOC_HISTORY_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_TEMPLATE + "-history-write"; + public static final String IOC_HISTORY_INDEX_PATTERN = "<." + IOC_INDEX_NAME_BASE + "-history-{now/d{yyyy.MM.dd.hh.mm.ss|UTC}}-1>"; + private Client client; private ClusterService clusterService; @@ -54,44 +69,49 @@ public IocService(Client client, ClusterService clusterService) { * @param index The index to evaluate. * @return TRUE if the index is an IOC-related system index, and exists; else returns FALSE. */ - public boolean hasIocSystemIndex(String index) { + public boolean feedIndexExists(String index) { return index.startsWith(IOC_INDEX_NAME_BASE) && this.clusterService.state().routingTable().hasIndex(index); } - public void initSystemIndexes(String index, ActionListener listener) { - if (!hasIocSystemIndex(index)) { - var indexRequest = new CreateIndexRequest(index) - // TODO hurneyt finalize mappings once IOC data model PR is merged -// .mapping(iocMappings()) + public static String getFeedConfigIndexName(String feedSourceConfigId) { + return IOC_INDEX_NAME_TEMPLATE.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); + } + + // TODO hurneyt change ActionResponse to more specific response once it's available + public String initFeedIndex(String feedSourceConfigId, ActionListener listener) { + String feedIndexName = getFeedConfigIndexName(feedSourceConfigId); + if (!feedIndexExists(feedIndexName)) { + var indexRequest = new CreateIndexRequest(feedIndexName) + .mapping(iocIndexMapping()) .settings(Settings.builder().put("index.hidden", true).build()); ((AdminClient) client).indices().create(indexRequest, new ActionListener<>() { @Override public void onResponse(CreateIndexResponse createIndexResponse) { - // TODO should this be info, or debug level? - log.info("Created system index {}", index); + log.info("Created system index {}", feedIndexName); } @Override public void onFailure(Exception e) { - log.error("Failed to create system index {}", index); + log.error("Failed to create system index {}", feedIndexName); listener.onFailure(e); } }); } + return feedIndexName; } - public void indexIocs(List allIocs, ActionListener listener) throws IOException { + public void indexIocs(String feedSourceConfigId, List iocs, ActionListener listener) throws IOException { // TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + String feedIndexName = initFeedIndex(feedSourceConfigId, listener); + List bulkRequestList = new ArrayList<>(); BulkRequest bulkRequest = new BulkRequest(); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - for (IocDao ioc : allIocs) { - initSystemIndexes(ioc.getType().getSystemIndexName(), listener); - - IndexRequest indexRequest = new IndexRequest(ioc.getType().getSystemIndexName()) + for (IOC ioc : iocs) { + IndexRequest indexRequest = new IndexRequest(feedIndexName) .opType(DocWriteRequest.OpType.INDEX) .source(ioc.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); bulkRequest.add(indexRequest); @@ -99,9 +119,9 @@ public void indexIocs(List allIocs, ActionListener bulkResponseListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { @@ -110,22 +130,33 @@ public void indexIocs(List allIocs, ActionListener client.bulk(req, bulkResponseListener)); + listener.onResponse(new FetchIocsActionResponse(iocs)); } catch (OpenSearchException e) { log.error("Failed to save IOCs.", e); } } } + public String iocIndexMapping() { + String iocMappingFile = "mappings/ioc_mapping.json"; + try (InputStream is = getClass().getClassLoader().getResourceAsStream(iocMappingFile)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(is, out); + return out.toString(StandardCharsets.UTF_8); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("Failed to load ioc_mapping.json file [{}]", iocMappingFile), e); + throw new IllegalStateException("Failed to load ioc_mapping.json file [" + iocMappingFile + "]", e); + } + } } diff --git a/src/main/resources/mappings/ioc_mapping.json b/src/main/resources/mappings/ioc_mapping.json new file mode 100644 index 000000000..a41c31251 --- /dev/null +++ b/src/main/resources/mappings/ioc_mapping.json @@ -0,0 +1,54 @@ +{ + "_meta" : { + "schema_version": 1 + }, + "properties": { + "ioc": { + "type": "nested", + "dynamic": "false", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + }, + "specVersion": { + "type": "keyword" + }, + "created": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "modified": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "description": { + "type": "text" + }, + "labels": { + "type": "keyword" + }, + "feedId": { + "type": "keyword" + } + } + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index c6874d915..9fd316ad9 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -29,7 +29,7 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.IoCMatch; -import org.opensearch.securityanalytics.model.IocDao; +import org.opensearch.securityanalytics.model.IOC; import org.opensearch.securityanalytics.model.IocDto; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.test.OpenSearchTestCase; @@ -2716,8 +2716,8 @@ public static XContentBuilder builder() throws IOException { return XContentBuilder.builder(XContentType.JSON.xContent()); } - public static IocDao randomIocDao() { - return randomIocDao( + public static IOC randomIOC() { + return randomIOC( null, null, null, @@ -2732,10 +2732,10 @@ public static IocDao randomIocDao() { ); } - public static IocDao randomIocDao( + public static IOC randomIOC( String id, String name, - IocDao.IocType type, + IOC.IocType type, String value, String severity, String specVersion, @@ -2752,7 +2752,7 @@ public static IocDao randomIocDao( name = randomString(); } if (type == null) { - type = IocDao.IocType.values()[randomInt(IocDao.IocType.values().length - 1)]; + type = IOC.IocType.values()[randomInt(IOC.IocType.values().length - 1)]; } if (value == null) { value = randomString(); @@ -2780,7 +2780,7 @@ public static IocDao randomIocDao( if (feedId == null) { feedId = randomString(); } - return new IocDao( + return new IOC( id, name, type, @@ -2796,13 +2796,13 @@ public static IocDao randomIocDao( } public static IocDto randomIocDto() { - return new IocDto(randomIocDao()); + return new IocDto(randomIOC()); } public static IocDto randomIocDto( String id, String name, - IocDao.IocType type, + IOC.IocType type, String value, String severity, String specVersion, @@ -2812,7 +2812,7 @@ public static IocDto randomIocDto( List labels, String feedId ) { - return new IocDto(randomIocDao( + return new IocDto(randomIOC( id, name, type, diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java b/src/test/java/org/opensearch/securityanalytics/model/IOCTests.java similarity index 76% rename from src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java rename to src/test/java/org/opensearch/securityanalytics/model/IOCTests.java index a441851d4..4cda68b3a 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/IOCTests.java @@ -11,37 +11,37 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.TestHelpers; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import static org.opensearch.securityanalytics.TestHelpers.parser; -import static org.opensearch.securityanalytics.TestHelpers.randomIocDao; -public class IocDaoTests extends OpenSearchTestCase { +public class IOCTests extends OpenSearchTestCase { public void testAsStream() throws IOException { - IocDao ioc = randomIocDao(); + IOC ioc = TestHelpers.randomIOC(); BytesStreamOutput out = new BytesStreamOutput(); ioc.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IocDao newIoc = new IocDao(sin); - assertEqualIocDaos(ioc, newIoc); + IOC newIoc = new IOC(sin); + assertEqualIOCs(ioc, newIoc); } public void testParseFunction() throws IOException { - IocDao ioc = randomIocDao(); + IOC ioc = TestHelpers.randomIOC(); String json = toJsonString(ioc); - IocDao newIoc = IocDao.parse(parser(json), ioc.getId()); - assertEqualIocDaos(ioc, newIoc); + IOC newIoc = IOC.parse(parser(json), ioc.getId()); + assertEqualIOCs(ioc, newIoc); } - private String toJsonString(IocDao ioc) throws IOException { + private String toJsonString(IOC ioc) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); return BytesReference.bytes(builder).utf8ToString(); } - public static void assertEqualIocDaos(IocDao ioc, IocDao newIoc) { + public static void assertEqualIOCs(IOC ioc, IOC newIoc) { assertEquals(ioc.getId(), newIoc.getId()); assertEquals(ioc.getName(), newIoc.getName()); assertEquals(ioc.getValue(), newIoc.getValue()); diff --git a/src/test/java/org/opensearch/securityanalytics/services/IocServiceIT.java b/src/test/java/org/opensearch/securityanalytics/services/IocServiceIT.java index 1088b1264..7664c9d04 100644 --- a/src/test/java/org/opensearch/securityanalytics/services/IocServiceIT.java +++ b/src/test/java/org/opensearch/securityanalytics/services/IocServiceIT.java @@ -14,13 +14,14 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.TestHelpers; import org.opensearch.securityanalytics.action.FetchIocsActionResponse; -import org.opensearch.securityanalytics.model.IocDao; -import org.opensearch.securityanalytics.model.IocDaoTests; +import org.opensearch.securityanalytics.model.IOC; +import org.opensearch.securityanalytics.model.IOCTests; import org.opensearch.securityanalytics.model.IocDto; import org.opensearch.test.OpenSearchIntegTestCase; @@ -28,21 +29,21 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Locale; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_ALL_INDEX_PATTERN; -import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_IP_INDEX_NAME; +import static org.opensearch.securityanalytics.services.IocService.IOC_ALL_INDEX_PATTERN; public class IocServiceIT extends OpenSearchIntegTestCase { private IocService service; + private String testFeedSourceConfigId; private String testIndex; @Before private void beforeTest() { service = new IocService(client(), clusterService()); + testFeedSourceConfigId = null; testIndex = null; } @@ -55,39 +56,43 @@ private void afterTest() throws ExecutionException, InterruptedException { public void test_hasIocSystemIndex_returnsFalse_whenIndexNotCreated() throws ExecutionException, InterruptedException { // Confirm index doesn't exist before running test case - testIndex = IOC_IP_INDEX_NAME; + testFeedSourceConfigId = randomAlphaOfLength(5); + testIndex = IocService.getFeedConfigIndexName(testFeedSourceConfigId); ClusterHealthResponse clusterHealthResponse = client().admin().cluster().health(new ClusterHealthRequest()).get(); assertFalse(clusterHealthResponse.getIndices().containsKey(testIndex)); // Run test case - assertFalse(service.hasIocSystemIndex(testIndex)); + assertFalse(service.feedIndexExists(testIndex)); } public void test_hasIocSystemIndex_returnsFalse_withInvalidIndex() throws ExecutionException, InterruptedException { // Create test index - testIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + testFeedSourceConfigId = randomAlphaOfLength(5); + testIndex = IocService.getFeedConfigIndexName(testFeedSourceConfigId); client().admin().indices().create(new CreateIndexRequest(testIndex)).get(); // Run test case - assertFalse(service.hasIocSystemIndex(testIndex)); + assertFalse(service.feedIndexExists(testIndex)); } public void test_hasIocSystemIndex_returnsTrue_whenIndexExists() throws ExecutionException, InterruptedException { // Create test index - testIndex = IOC_IP_INDEX_NAME; + testFeedSourceConfigId = randomAlphaOfLength(5); + testIndex = IocService.getFeedConfigIndexName(testFeedSourceConfigId); client().admin().indices().create(new CreateIndexRequest(testIndex)).get(); // Run test case - assertTrue(service.hasIocSystemIndex(testIndex)); + assertTrue(service.feedIndexExists(testIndex)); } public void test_initSystemIndexes_createsIndexes() { // Confirm index doesn't exist - testIndex = IOC_IP_INDEX_NAME; - assertFalse(service.hasIocSystemIndex(testIndex)); + testFeedSourceConfigId = randomAlphaOfLength(5); + testIndex = IocService.getFeedConfigIndexName(testFeedSourceConfigId); + assertFalse(service.feedIndexExists(testIndex)); // Run test case - service.initSystemIndexes(testIndex, new ActionListener<>() { + service.initFeedIndex(testIndex, new ActionListener<>() { @Override public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) {} @@ -96,17 +101,18 @@ public void onFailure(Exception e) { fail(String.format("Creation of %s should not fail: %s", testIndex, e)); } }); - assertTrue(service.hasIocSystemIndex(testIndex)); + assertTrue(service.feedIndexExists(testIndex)); } public void test_indexIocs_ingestsIocsCorrectly() throws IOException { // Prepare test IOCs - List iocs = IntStream.range(0, randomInt()) - .mapToObj(i -> TestHelpers.randomIocDao()) + testFeedSourceConfigId = randomAlphaOfLength(5); + List iocs = IntStream.range(0, randomInt()) + .mapToObj(i -> TestHelpers.randomIOC()) .collect(Collectors.toList()); // Run test case - service.indexIocs(iocs, new ActionListener<>() { + service.indexIocs(testFeedSourceConfigId, iocs, new ActionListener<>() { @Override public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) { // Confirm expected number of IOCs in response @@ -123,10 +129,10 @@ public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) { assertEquals(iocs.size(), searchResponse.getHits().getHits().length); // Parse hits to IOCs - List iocHits = Collections.emptyList(); + List iocHits = Collections.emptyList(); for (SearchHit ioc : searchResponse.getHits()) { try { - iocHits.add(IocDao.parse(TestHelpers.parser(ioc.getSourceAsString()), null)); + iocHits.add(IOC.parse(TestHelpers.parser(ioc.getSourceAsString()), null)); } catch (IOException e) { fail(String.format("Failed to parse IOC hit: %s", e)); } @@ -136,14 +142,14 @@ public void onResponse(FetchIocsActionResponse fetchIocsActionResponse) { assertEquals(iocs.size(), iocHits.size()); // Sort IOCs for comparison - iocs.sort(Comparator.comparing(IocDao::getId)); + iocs.sort(Comparator.comparing(IOC::getId)); fetchIocsActionResponse.getIocs().sort(Comparator.comparing(IocDto::getId)); - iocHits.sort(Comparator.comparing(IocDao::getId)); + iocHits.sort(Comparator.comparing(IOC::getId)); // Confirm IOCs are equal for (int i = 0; i < iocs.size(); i++) { assertEqualIocs(iocs.get(i), fetchIocsActionResponse.getIocs().get(i)); - IocDaoTests.assertEqualIocDaos(iocs.get(i), iocHits.get(i)); + IOCTests.assertEqualIOCs(iocs.get(i), iocHits.get(i)); } } catch (InterruptedException | ExecutionException e) { fail(String.format("IOC_ALL_INDEX_PATTERN search failed: %s", e)); @@ -157,16 +163,16 @@ public void onFailure(Exception e) { }); } - private void assertEqualIocs(IocDao iocDao, IocDto iocDto) { - assertEquals(iocDao.getId(), iocDto.getId()); - assertEquals(iocDao.getName(), iocDto.getName()); - assertEquals(iocDao.getValue(), iocDto.getValue()); - assertEquals(iocDao.getSeverity(), iocDto.getSeverity()); - assertEquals(iocDao.getSpecVersion(), iocDto.getSpecVersion()); - assertEquals(iocDao.getCreated(), iocDto.getCreated()); - assertEquals(iocDao.getModified(), iocDto.getModified()); - assertEquals(iocDao.getDescription(), iocDto.getDescription()); - assertEquals(iocDao.getLabels(), iocDto.getLabels()); - assertEquals(iocDao.getFeedId(), iocDto.getFeedId()); + private void assertEqualIocs(IOC ioc, IocDto iocDto) { + assertEquals(ioc.getId(), iocDto.getId()); + assertEquals(ioc.getName(), iocDto.getName()); + assertEquals(ioc.getValue(), iocDto.getValue()); + assertEquals(ioc.getSeverity(), iocDto.getSeverity()); + assertEquals(ioc.getSpecVersion(), iocDto.getSpecVersion()); + assertEquals(ioc.getCreated(), iocDto.getCreated()); + assertEquals(ioc.getModified(), iocDto.getModified()); + assertEquals(ioc.getDescription(), iocDto.getDescription()); + assertEquals(ioc.getLabels(), iocDto.getLabels()); + assertEquals(ioc.getFeedId(), iocDto.getFeedId()); } }