From ed41a7e13dd88be0d9fd7ff5526d67243f0386b7 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 23 May 2024 17:51:00 -0700 Subject: [PATCH 1/7] create tif source config api implementation Signed-off-by: Joanne Wang --- .../SecurityAnalyticsPlugin.java | 46 +++- .../action/SAIndexTIFSourceConfigAction.java | 22 ++ .../action/SAIndexTIFSourceConfigRequest.java | 96 ++++++++ .../SAIndexTIFSourceConfigResponse.java | 214 ++++++++++++++++++ .../threatIntel/dao/SATIFSourceConfigDao.java | 142 ++++++++++++ .../threatIntel/model/SATIFSourceConfig.java | 72 ++++-- .../model/SATIFSourceConfigDto.java | 108 +++++---- .../threatIntel/model/TIFJobParameter.java | 1 - .../resthandler/RestIndexTIFConfigAction.java | 91 ++++++++ .../sacommons/IndexTIFSourceConfigAction.java | 11 + .../IndexTIFSourceConfigRequest.java | 14 ++ .../IndexTIFSourceConfigResponse.java | 134 +++++++++++ .../sacommons/TIFSourceConfig.java | 16 +- .../sacommons/TIFSourceConfigDao.java | 11 + .../sacommons/TIFSourceConfigDto.java | 16 +- .../service/SATIFSourceConfigService.java | 111 +++++++++ .../TransportIndexTIFSourceConfigAction.java | 143 ++++++++++++ .../transport/TransportPutTIFJobAction.java | 1 - .../mappings/threat_intel_job_mapping.json | 20 +- .../SecurityAnalyticsRestTestCase.java | 9 + .../SATIFSourceConfigRestApiIT.java | 76 +++++++ 21 files changed, 1267 insertions(+), 87 deletions(-) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDao.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java create mode 100644 src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 652b438df..de28cbfbc 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -30,6 +30,8 @@ import org.opensearch.commons.alerting.action.AlertingActions; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; @@ -63,13 +65,18 @@ import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.*; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFConfigAction; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; @@ -87,6 +94,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { @@ -103,9 +111,11 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; - + public static final String TIF_CONFIG_URI = PLUGINS_BASE_URI + "/tif"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; + public static final String JOB_TYPE = "opensearch_sap_job"; + public static final Map TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true); private CorrelationRuleIndices correlationRuleIndices; @@ -129,6 +139,9 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private BuiltinLogTypeLoader builtinLogTypeLoader; private LogTypeService logTypeService; + + private SATIFSourceConfigDao satifSourceConfigDao; + @Override public Collection getSystemIndexDescriptors(Settings settings){ return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); @@ -165,13 +178,16 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); + satifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool); + SATIFSourceConfigService satifSourceConfigService = new SATIFSourceConfigService(satifSourceConfigDao, threatIntelLockService); + TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService); + tifJobUpdateService, tifJobParameterService, threatIntelLockService, satifSourceConfigDao, satifSourceConfigService); } @Override @@ -211,13 +227,14 @@ public List getRestHandlers(Settings settings, new RestSearchCorrelationRuleAction(), new RestIndexCustomLogTypeAction(), new RestSearchCustomLogTypeAction(), - new RestDeleteCustomLogTypeAction() + new RestDeleteCustomLogTypeAction(), + new RestIndexTIFConfigAction() ); } @Override public String getJobType() { - return "opensearch_sap_job"; + return JOB_TYPE; } @Override @@ -232,7 +249,21 @@ public ScheduledJobRunner getJobRunner() { @Override public ScheduledJobParser getJobParser() { - return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); + return (xcp, id, jobDocVersion) -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + return SATIFSourceConfig.parse(xcp, id, null); + default: + log.warn("Unsupported document was indexed"); + xcp.skipChildren(); + } + } + return null; + }; } @Override @@ -332,7 +363,8 @@ public List> getSettings() { new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), - new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class) + new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), + new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..1b4acd80e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Threat intel tif job creation action + */ +public class SAIndexTIFSourceConfigAction extends ActionType { + + public static final SAIndexTIFSourceConfigAction INSTANCE = new SAIndexTIFSourceConfigAction(); + public static final String NAME = INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + private SAIndexTIFSourceConfigAction() { + super(NAME, SAIndexTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..073b2a4b8 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigRequest; + +import java.io.IOException; +import java.sql.Time; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +/** + * Threat intel feed config creation request + */ +public class SAIndexTIFSourceConfigRequest extends ActionRequest implements IndexTIFSourceConfigRequest { + private static final ParameterValidator VALIDATOR = new ParameterValidator(); + private String tifSourceConfigId; + private final WriteRequest.RefreshPolicy refreshPolicy; + private final RestRequest.Method method; + private SATIFSourceConfigDto satifSourceConfigDto; + + public SAIndexTIFSourceConfigRequest(String tifSourceConfigId, + WriteRequest.RefreshPolicy refreshPolicy, + RestRequest.Method method, + SATIFSourceConfigDto satifSourceConfigDto) { + super(); + this.tifSourceConfigId = tifSourceConfigId; + this.refreshPolicy = refreshPolicy; + this.method = method; + this.satifSourceConfigDto = satifSourceConfigDto; + } + + public SAIndexTIFSourceConfigRequest(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + WriteRequest.RefreshPolicy.readFrom(sin), // refresh policy + sin.readEnum(RestRequest.Method.class), // method + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(tifSourceConfigId); + refreshPolicy.writeTo(out); + out.writeEnum(method); + satifSourceConfigDto.writeTo(out); + } + + @Override + public String getTIFConfigId() { + return tifSourceConfigId; + } + + public void setTIFConfigId(String tifConfigId) { + this.tifSourceConfigId = tifConfigId; + } + + @Override + public SATIFSourceConfigDto getTIFConfigDto() { + return satifSourceConfigDto; + } + + public void setTIFConfigDto(SATIFSourceConfigDto saTifConfigDto) { + this.satifSourceConfigDto = saTifConfigDto; + } + + public WriteRequest.RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + List errorMsgs = VALIDATOR.validateTIFJobName(satifSourceConfigDto.getName()); + if (errorMsgs.isEmpty() == false) { + errorMsgs.forEach(errors::addValidationError); + } + return errors.validationErrors().isEmpty() ? null : errors; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..457da54ef --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -0,0 +1,214 @@ +///* +// * Copyright OpenSearch Contributors +// * SPDX-License-Identifier: Apache-2.0 +// */ +//package org.opensearch.securityanalytics.threatIntel.action; +// +//import org.opensearch.core.action.ActionResponse; +//import org.opensearch.core.common.io.stream.StreamInput; +//import org.opensearch.core.common.io.stream.StreamOutput; +//import org.opensearch.core.xcontent.ToXContentObject; +//import org.opensearch.core.xcontent.XContentBuilder; +//import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +//import org.opensearch.securityanalytics.threatIntel.model.SATIFConfig; +// +//import java.io.IOException; +//import java.time.Instant; +//import java.util.List; +// +//import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +//import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; +// +//public class IndexTIFConfigResponse extends ActionResponse implements ToXContentObject { +// private String id; +// private Long version; +// private String feedFormat; +// private String feedName; +// private TIFJobState state; +// private Integer numFindings; +// private Integer numIOCs; +// private Instant lastActivatedTime; +// private Instant lastRefreshedTime; +// private String refreshInterval; +// +//// private Source source; +// private String createdByUser; +// private String feedType; +// private Boolean licenseRequired; +// private Integer numScansConfigured; +// private List iocTypes; +// +// public IndexTIFConfigResponse(String feedId, Long version, String feedFormat, String feedName, TIFJobState state, Integer numFindings, +// Integer numIOCs, Instant lastActivatedTime, Instant lastRefreshedTime, String refreshInterval, String createdByUser, +// String feedType, Boolean licenseRequired, Integer numScansConfigured, List iocTypes) { +// super(); +// this.id = feedId; +// this.version = version; +// this.feedFormat = feedFormat; +// this.feedName = feedName; +// this.state = state; +// this.numFindings = numFindings; +// this.numIOCs = numIOCs; +// this.lastActivatedTime = lastActivatedTime; +// this.lastRefreshedTime = lastRefreshedTime; +// this.refreshInterval = refreshInterval; +// this.createdByUser = createdByUser; +// this.feedType = feedType; +// this.licenseRequired = licenseRequired; +// this.numScansConfigured = numScansConfigured; +// this.iocTypes = iocTypes; +// } +// +// public IndexTIFConfigResponse(StreamInput sin) throws IOException { +// this.id = sin.readString(); +// this.version = sin.readLong(); +// this.feedFormat = sin.readString(); +// this.feedName = sin.readString(); +// this.state = TIFJobState.valueOf(sin.readString()); +// this.numFindings = sin.readInt(); +// this.numIOCs = sin.readInt(); +// this.lastActivatedTime = sin.readInstant(); +// this.lastRefreshedTime = sin.readInstant(); +// this.refreshInterval = sin.readString(); +// this.createdByUser = sin.readString(); +// this.feedType = sin.readString(); +// this.licenseRequired = sin.readBoolean(); +// this.numScansConfigured = sin.readInt(); +// this.iocTypes = sin.readStringList(); +// } +// +// @Override +// public void writeTo(StreamOutput out) throws IOException { +// out.writeString(id); +// out.writeString(feedFormat); +// out.writeString(feedName); +// out.writeString(state.name()); +// out.writeInt(numFindings); +// out.writeInt(numIOCs); +// out.writeInstant(lastActivatedTime); +// out.writeInstant(lastRefreshedTime); +// out.writeString(refreshInterval); +// out.writeString(createdByUser); +// out.writeString(feedType); +// out.writeBoolean(licenseRequired); +// out.writeInt(numScansConfigured); +// out.writeStringCollection(iocTypes); +// } +// +// @Override +// public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { +// builder.startObject() +// .field(_ID, id) +// .field(_VERSION, version); +// +// builder.startObject("tif_config") +// .field(SATIFConfig.FEED_FORMAT_FIELD, feedFormat) +// .field(SATIFConfig.FEED_NAME_FIELD, feedName) +// .field(SATIFConfig.STATE_FIELD, state) +// .field("num_findings", numFindings) +// .field("num_iocs", numIOCs) +// .field(SATIFConfig.ENABLED_TIME_FIELD, lastActivatedTime) +// .field(SATIFConfig.RefreshStats.LAST_REFRESHED_TIME_FIELD, lastRefreshedTime) +// .field("refresh_interval", refreshInterval) +// .field(SATIFConfig.CREATED_BY_USER_FIELD, createdByUser) +// .field(SATIFConfig.FEED_FORMAT_FIELD, feedType) +// .field("license_required", licenseRequired) +// .field("num_scans_configured", numScansConfigured) +// .field("ioc_types", iocTypes) +// .endObject(); +// return builder.endObject(); +// } +// +//} + +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SAIndexTIFSourceConfigResponse extends ActionResponse implements ToXContentObject, IndexTIFSourceConfigResponse { + private final String tifConfigId; + private final Long version; + private final RestStatus status; + private final SATIFSourceConfigDto saTIFConfigDto; + + public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto tifConfig) { + super(); + this.tifConfigId = id; + this.version = version; + this.status = status; + this.saTIFConfigDto = tifConfig; + } + + public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + sin.readLong(), // version + sin.readEnum(RestStatus.class), // status + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(tifConfigId); + out.writeLong(version); + out.writeEnum(status); + saTIFConfigDto.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, tifConfigId) + .field(_VERSION, version); + + builder.startObject("tif_config") + .field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, saTIFConfigDto.getFeedFormat()) + .field(SATIFSourceConfigDto.FEED_NAME_FIELD, saTIFConfigDto.getName()) + .field(SATIFSourceConfigDto.FEED_TYPE_FIELD, saTIFConfigDto.getFeedType()) + .field(SATIFSourceConfigDto.STATE_FIELD, saTIFConfigDto.getState()) + .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTIFConfigDto.getEnabledTime()) + .field(SATIFSourceConfigDto.ENABLED_FIELD, saTIFConfigDto.isEnabled()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTIFConfigDto.getLastRefreshedTime()) + .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTIFConfigDto.getSchedule()) + // source + .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTIFConfigDto.getCreatedByUser()) + .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTIFConfigDto.getIocTypes()) + .endObject(); + + return builder.endObject(); + } + @Override + public String getTIFConfigId() { + return tifConfigId; + } + @Override + public Long getVersion() { + return version; + } + @Override + public TIFSourceConfigDto getTIFConfigDto() { + return saTIFConfigDto; + } + public RestStatus getStatus() { + return status; + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java new file mode 100644 index 000000000..5078bc85b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java @@ -0,0 +1,142 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.threadpool.ThreadPool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +/** + * CRUD for threat intel feeds source config object + */ +public class SATIFSourceConfigDao { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigDao.class); + private final Client client; + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final ThreadPool threadPool; + + + public SATIFSourceConfigDao(final Client client, final ClusterService clusterService, ThreadPool threadPool) { + this.client = client; + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.threadPool = threadPool; + } + + public void indexTIFSourceConfig(SATIFSourceConfig satifSourceConfig, + TimeValue indexTimeout, + WriteRequest.RefreshPolicy refreshPolicy, + final ActionListener actionListener) throws Exception { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(refreshPolicy) + .source(satifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + log.debug("Indexing tif source config"); + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + log.debug("TIF source config indexed success."); + satifSourceConfig.setId(response.getId()); + actionListener.onResponse(satifSourceConfig); + } + @Override + public void onFailure(Exception e) { + throw new SecurityAnalyticsException("Exception saving the tif source config in index", RestStatus.INTERNAL_SERVER_ERROR, e); + } + }); + } + + public ThreadPool getThreadPool() { + return threadPool; + } + + + // Get the job config index mapping + private String getIndexMapping() { + try { + try (InputStream is = SATIFSourceConfigDao.class.getResourceAsStream("/mappings/threat_intel_job_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Runtime exception when getting the threat intel index mapping", e); + throw new SecurityAnalyticsException("Runtime exception when getting the threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + // Create Threat intel config index + /** + * Index name: .opensearch-sap--job + * Mapping: /mappings/threat_intel_job_mapping.json + * + * @param stepListener setup listener + */ + public void createJobIndexIfNotExists(final StepListener stepListener) { + // check if job index exists + if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { + stepListener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { + @Override + public void onResponse(final CreateIndexResponse createIndexResponse) { + log.debug("Job index created"); + stepListener.onResponse(null); + } + + @Override + public void onFailure(final Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + log.info("index[{}] already exist", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + log.error("Failed to create security analytics threat intel job index", e); + stepListener.onFailure(e); + } + })); + } + + // Common utils interface class + IndexTIFSourceConfigResponse indexTIFConfig(SAIndexTIFSourceConfigRequest request, ActionListener listener) { + return null; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java index 46f576b4e..56cfb7fa2 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.UUIDs; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -19,7 +18,6 @@ import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.jobscheduler.spi.schedule.Schedule; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; @@ -27,6 +25,8 @@ import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -37,14 +37,13 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ private static final Logger log = LogManager.getLogger(SATIFSourceConfig.class); - /** * Prefix of indices having threatIntel data */ public static final String THREAT_INTEL_DATA_INDEX_NAME_PREFIX = ".opensearch-sap-threat-intel"; + public static final String FEED_SOURCE_CONFIG_FIELD = "feed_source_config"; public static final String NO_ID = ""; - public static final String ID_FIELD = "id"; public static final Long NO_VERSION = 1L; public static final String VERSION_FIELD = "version"; @@ -63,6 +62,7 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; public static final String ENABLED_FIELD = "enabled"; public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + public static final String IOC_TYPES_FIELD = "ioc_types"; private String id; private Long version; @@ -75,17 +75,18 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ // private Source source; TODO: create Source Object private Instant enabledTime; private Instant lastUpdateTime; - private Schedule schedule; + private IntervalSchedule schedule; private TIFJobState state; public String refreshType; public Instant lastRefreshedTime; public String lastRefreshedUser; private Boolean isEnabled; private Map iocMapStore; + private List iocTypes; public SATIFSourceConfig(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, - Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, - Boolean isEnabled, Map iocMapStore) { + Instant enabledTime, Instant lastUpdateTime, IntervalSchedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore, List iocTypes) { this.id = id != null ? id : NO_ID; this.version = version != null ? version : NO_VERSION; this.feedName = feedName; @@ -112,6 +113,7 @@ public SATIFSourceConfig(String id, Long version, String feedName, String feedFo this.lastRefreshedUser = lastRefreshedUser; this.isEnabled = isEnabled; this.iocMapStore = iocMapStore; + this.iocTypes = iocTypes; } public SATIFSourceConfig(StreamInput sin) throws IOException { @@ -131,7 +133,8 @@ public SATIFSourceConfig(StreamInput sin) throws IOException { sin.readOptionalInstant(), // last refreshed time sin.readOptionalString(), // last refreshed user sin.readBoolean(), // is enabled - sin.readMap() // ioc map store + sin.readMap(), // ioc map store + sin.readStringList() ); } @@ -152,17 +155,18 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalString(lastRefreshedUser == null? null : lastRefreshedUser); out.writeBoolean(isEnabled); out.writeMap(iocMapStore); + out.writeStringCollection(iocTypes); } @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject(); - builder.field(ID_FIELD, id); - builder.field(VERSION_FIELD, version); - builder.field(FEED_NAME_FIELD, feedName); - builder.field(FEED_FORMAT_FIELD, feedFormat); - builder.field(FEED_TYPE_FIELD, feedType.name()); - builder.field(CREATED_BY_USER_FIELD, createdByUser); + builder.startObject() + .startObject(FEED_SOURCE_CONFIG_FIELD) + .field(VERSION_FIELD, version) + .field(FEED_NAME_FIELD, feedName) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(FEED_TYPE_FIELD, feedType.name()) + .field(CREATED_BY_USER_FIELD, createdByUser); if (createdAt == null) { builder.nullField(CREATED_AT_FIELD); @@ -194,6 +198,8 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); builder.field(ENABLED_FIELD, isEnabled); builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); builder.endObject(); return builder; } @@ -213,21 +219,23 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio Instant createdAt = null; Instant enabledTime = null; Instant lastUpdateTime = null; - Schedule schedule = null; + IntervalSchedule schedule = null; TIFJobState state = null; String refreshType = null; Instant lastRefreshedTime = null; String lastRefreshedUser = null; Boolean isEnabled = null; Map iocMapStore = null; + List iocTypes = new ArrayList<>(); - xcp.nextToken(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); xcp.nextToken(); switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + break; case FEED_NAME_FIELD: feedName = xcp.text(); break; @@ -275,7 +283,7 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio } break; case SCHEDULE_FIELD: - schedule = ScheduleParser.parse(xcp); + schedule = (IntervalSchedule) ScheduleParser.parse(xcp); break; case STATE_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -312,9 +320,18 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio isEnabled = xcp.booleanValue(); break; case IOC_MAP_STORE_FIELD: - iocMapStore = xcp.map(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocMapStore = null; + } else { + iocMapStore = xcp.map(); + } + break; + case IOC_TYPES_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } break; - default: xcp.skipChildren(); } @@ -342,7 +359,8 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio lastRefreshedTime, lastRefreshedUser, isEnabled, - iocMapStore + iocMapStore, + iocTypes ); } @@ -424,10 +442,10 @@ public Instant getLastUpdateTime() { public void setLastUpdateTime(Instant lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } - public Schedule getSchedule() { + public IntervalSchedule getSchedule() { return this.schedule; } - public void setSchedule(Schedule schedule) { + public void setSchedule(IntervalSchedule schedule) { this.schedule = schedule; } public TIFJobState getState() { @@ -474,4 +492,12 @@ public Map getIocMapStore() { public void setIocMapStore(Map iocMapStore) { this.iocMapStore = iocMapStore; } + + public List getIocTypes() { + return iocTypes; + } + + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index c8344e5e1..c5eb1c6a8 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.UUIDs; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -18,7 +17,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; -import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; @@ -26,7 +25,9 @@ import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -37,9 +38,9 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou private static final Logger log = LogManager.getLogger(SATIFSourceConfigDto.class); + public static final String FEED_SOURCE_CONFIG_FIELD = "feed_source_config"; public static final String NO_ID = ""; - public static final String ID_FIELD = "id"; public static final Long NO_VERSION = 1L; public static final String VERSION_FIELD = "version"; @@ -58,6 +59,7 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; public static final String ENABLED_FIELD = "enabled"; public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + public static final String IOC_TYPES_FIELD = "ioc_types"; private String id; private Long version; @@ -70,13 +72,14 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou // private Source source; TODO: create Source Object private Instant enabledTime; private Instant lastUpdateTime; - private Schedule schedule; + private IntervalSchedule schedule; private TIFJobState state; public String refreshType; public Instant lastRefreshedTime; public String lastRefreshedUser; private Boolean isEnabled; private Map iocMapStore; + private List iocTypes; public SATIFSourceConfigDto(SATIFSourceConfig saTIFSourceConfig) { this.id = saTIFSourceConfig.getId(); @@ -95,11 +98,12 @@ public SATIFSourceConfigDto(SATIFSourceConfig saTIFSourceConfig) { this.lastRefreshedUser = saTIFSourceConfig.getLastRefreshedUser(); this.isEnabled = saTIFSourceConfig.isEnabled();; this.iocMapStore = saTIFSourceConfig.getIocMapStore(); + this.iocTypes = saTIFSourceConfig.getIocTypes(); } public SATIFSourceConfigDto(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, - Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, - Boolean isEnabled, Map iocMapStore) { + Instant enabledTime, Instant lastUpdateTime, IntervalSchedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore, List iocTypes) { this.id = id != null ? id : NO_ID; this.version = version != null ? version : NO_VERSION; this.feedName = feedName; @@ -126,6 +130,7 @@ public SATIFSourceConfigDto(String id, Long version, String feedName, String fee this.lastRefreshedUser = lastRefreshedUser; this.isEnabled = isEnabled; this.iocMapStore = (this.iocMapStore == null) ? new HashMap<>() : iocMapStore; + this.iocTypes = iocTypes; } public SATIFSourceConfigDto(StreamInput sin) throws IOException { @@ -149,17 +154,18 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalString(lastRefreshedUser == null? null : lastRefreshedUser); out.writeBoolean(isEnabled); out.writeMap(iocMapStore); + out.writeStringCollection(iocTypes); } @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject(); - builder.field(ID_FIELD, id); - builder.field(VERSION_FIELD, version); - builder.field(FEED_NAME_FIELD, feedName); - builder.field(FEED_FORMAT_FIELD, feedFormat); - builder.field(FEED_TYPE_FIELD, feedType); - builder.field(CREATED_BY_USER_FIELD, createdByUser); + builder.startObject() + .startObject(FEED_SOURCE_CONFIG_FIELD) + .field(VERSION_FIELD, version) + .field(FEED_NAME_FIELD, feedName) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(FEED_TYPE_FIELD, feedType.name()) + .field(CREATED_BY_USER_FIELD, createdByUser); if (createdAt == null) { builder.nullField(CREATED_AT_FIELD); @@ -181,30 +187,19 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(SCHEDULE_FIELD, schedule); builder.field(STATE_FIELD, state.name()); - - if (refreshType == null) { - builder.nullField(REFRESH_TYPE_FIELD); - } else { - builder.field(REFRESH_TYPE_FIELD, refreshType); - } - + builder.field(REFRESH_TYPE_FIELD, refreshType); if (lastRefreshedTime == null) { builder.nullField(LAST_REFRESHED_TIME_FIELD); } else { builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); } - - if (lastRefreshedUser == null) { - builder.nullField(LAST_REFRESHED_USER_FIELD); - } else { - builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); - } builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); builder.field(ENABLED_FIELD, isEnabled); builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); builder.endObject(); - return builder; } @@ -223,21 +218,22 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver Instant createdAt = null; Instant enabledTime = null; Instant lastUpdateTime = null; - Schedule schedule = null; + IntervalSchedule schedule = null; TIFJobState state = null; String refreshType = null; Instant lastRefreshedTime = null; String lastRefreshedUser = null; Boolean isEnabled = null; - Map iocMapStore = new HashMap<>(); + Map iocMapStore = null; + List iocTypes = null; - xcp.nextToken(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); xcp.nextToken(); - switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + break; case FEED_NAME_FIELD: feedName = xcp.text(); break; @@ -248,7 +244,11 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver feedType = toFeedType(xcp.text()); break; case CREATED_BY_USER_FIELD: - createdByUser = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdByUser = null; + } else { + createdByUser = xcp.text(); + } break; case CREATED_AT_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -281,7 +281,7 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver } break; case SCHEDULE_FIELD: - schedule = ScheduleParser.parse(xcp); + schedule = (IntervalSchedule) ScheduleParser.parse(xcp); break; case STATE_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -291,7 +291,11 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver } break; case REFRESH_TYPE_FIELD: - refreshType = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + refreshType = null; + } else { + refreshType = xcp.text(); + } break; case LAST_REFRESHED_TIME_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -304,15 +308,29 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver } break; case LAST_REFRESHED_USER_FIELD: - lastRefreshedUser = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedUser = null; + } else { + lastRefreshedUser = xcp.text(); + } break; case ENABLED_FIELD: isEnabled = xcp.booleanValue(); break; case IOC_MAP_STORE_FIELD: - iocMapStore = xcp.map(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocMapStore = null; + } else { + iocMapStore = xcp.map(); + } + break; + case IOC_TYPES_FIELD: + iocTypes = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } break; - default: xcp.skipChildren(); } @@ -340,7 +358,8 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver lastRefreshedTime, lastRefreshedUser, isEnabled, - iocMapStore + iocMapStore, + iocTypes ); } @@ -419,10 +438,10 @@ public Instant getLastUpdateTime() { public void setLastUpdateTime(Instant lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } - public Schedule getSchedule() { + public IntervalSchedule getSchedule() { return this.schedule; } - public void setSchedule(Schedule schedule) { + public void setSchedule(IntervalSchedule schedule) { this.schedule = schedule; } public TIFJobState getState() { @@ -477,6 +496,15 @@ public Map getIocMapStore() { public void setIocMapStore(Map iocMapStore) { this.iocMapStore = iocMapStore; } + + public List getIocTypes() { + return iocTypes; + } + + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } + public static SATIFSourceConfigDto readFrom(StreamInput sin) throws IOException { return new SATIFSourceConfigDto(sin); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java index a964a1663..2fa5cb199 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java @@ -562,7 +562,6 @@ public static TIFJobParameter build(final PutTIFJobRequest request) { ChronoUnit.MINUTES ); return new TIFJobParameter(name, schedule); - } } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java new file mode 100644 index 000000000..ff6243cef --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.RestHandlerUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.NO_ID; + + +public class RestIndexTIFConfigAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestIndexTIFConfigAction.class); + @Override + public String getName() { + return "index_tif_config_action"; + } + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.TIF_CONFIG_URI), + new Route(RestRequest.Method.PUT, SecurityAnalyticsPlugin.TIF_CONFIG_URI + "/{tifConfigId}") + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.TIF_CONFIG_URI)); + + WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE; + if (request.hasParam(RestHandlerUtils.REFRESH)) { + refreshPolicy = WriteRequest.RefreshPolicy.parse(request.param(RestHandlerUtils.REFRESH)); + } + + String id = request.param("feed_id", null); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + SATIFSourceConfigDto tifConfig = SATIFSourceConfigDto.parse(xcp, id, null); + tifConfig.setLastUpdateTime(Instant.now()); + + SAIndexTIFSourceConfigRequest indexTIFConfigRequest = new SAIndexTIFSourceConfigRequest(id, refreshPolicy, request.method(), tifConfig); + return channel -> client.execute(SAIndexTIFSourceConfigAction.INSTANCE, indexTIFConfigRequest, indexTIFConfigResponse(channel, request.method())); + } + + private RestResponseListener indexTIFConfigResponse(RestChannel channel, RestRequest.Method restMethod) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(SAIndexTIFSourceConfigResponse response) throws Exception { + RestStatus returnStatus = RestStatus.CREATED; + if (restMethod == RestRequest.Method.PUT) { + returnStatus = RestStatus.OK; + } + + BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + + if (restMethod == RestRequest.Method.POST) { + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_CONFIG_URI, response.getTIFConfigId()); + restResponse.addHeader("Location", location); + } + + return restResponse; + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java new file mode 100644 index 000000000..ab358d453 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public class IndexTIFSourceConfigAction { + public static final String INDEX_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/security_analytics/tifConfig/write"; + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..db33575eb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +/** + * Threat intel feed config creation request interface + */ +public interface IndexTIFSourceConfigRequest { + String getTIFConfigId(); + TIFSourceConfigDto getTIFConfigDto(); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..8cbc1b025 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java @@ -0,0 +1,134 @@ +///* +// * Copyright OpenSearch Contributors +// * SPDX-License-Identifier: Apache-2.0 +// */ +//package org.opensearch.securityanalytics.threatIntel.action; +// +//import org.opensearch.core.action.ActionResponse; +//import org.opensearch.core.common.io.stream.StreamInput; +//import org.opensearch.core.common.io.stream.StreamOutput; +//import org.opensearch.core.xcontent.ToXContentObject; +//import org.opensearch.core.xcontent.XContentBuilder; +//import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +//import org.opensearch.securityanalytics.threatIntel.model.SATIFConfig; +// +//import java.io.IOException; +//import java.time.Instant; +//import java.util.List; +// +//import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +//import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; +// +//public class IndexTIFConfigResponse extends ActionResponse implements ToXContentObject { +// private String id; +// private Long version; +// private String feedFormat; +// private String feedName; +// private TIFJobState state; +// private Integer numFindings; +// private Integer numIOCs; +// private Instant lastActivatedTime; +// private Instant lastRefreshedTime; +// private String refreshInterval; +// +//// private Source source; +// private String createdByUser; +// private String feedType; +// private Boolean licenseRequired; +// private Integer numScansConfigured; +// private List iocTypes; +// +// public IndexTIFConfigResponse(String feedId, Long version, String feedFormat, String feedName, TIFJobState state, Integer numFindings, +// Integer numIOCs, Instant lastActivatedTime, Instant lastRefreshedTime, String refreshInterval, String createdByUser, +// String feedType, Boolean licenseRequired, Integer numScansConfigured, List iocTypes) { +// super(); +// this.id = feedId; +// this.version = version; +// this.feedFormat = feedFormat; +// this.feedName = feedName; +// this.state = state; +// this.numFindings = numFindings; +// this.numIOCs = numIOCs; +// this.lastActivatedTime = lastActivatedTime; +// this.lastRefreshedTime = lastRefreshedTime; +// this.refreshInterval = refreshInterval; +// this.createdByUser = createdByUser; +// this.feedType = feedType; +// this.licenseRequired = licenseRequired; +// this.numScansConfigured = numScansConfigured; +// this.iocTypes = iocTypes; +// } +// +// public IndexTIFConfigResponse(StreamInput sin) throws IOException { +// this.id = sin.readString(); +// this.version = sin.readLong(); +// this.feedFormat = sin.readString(); +// this.feedName = sin.readString(); +// this.state = TIFJobState.valueOf(sin.readString()); +// this.numFindings = sin.readInt(); +// this.numIOCs = sin.readInt(); +// this.lastActivatedTime = sin.readInstant(); +// this.lastRefreshedTime = sin.readInstant(); +// this.refreshInterval = sin.readString(); +// this.createdByUser = sin.readString(); +// this.feedType = sin.readString(); +// this.licenseRequired = sin.readBoolean(); +// this.numScansConfigured = sin.readInt(); +// this.iocTypes = sin.readStringList(); +// } +// +// @Override +// public void writeTo(StreamOutput out) throws IOException { +// out.writeString(id); +// out.writeString(feedFormat); +// out.writeString(feedName); +// out.writeString(state.name()); +// out.writeInt(numFindings); +// out.writeInt(numIOCs); +// out.writeInstant(lastActivatedTime); +// out.writeInstant(lastRefreshedTime); +// out.writeString(refreshInterval); +// out.writeString(createdByUser); +// out.writeString(feedType); +// out.writeBoolean(licenseRequired); +// out.writeInt(numScansConfigured); +// out.writeStringCollection(iocTypes); +// } +// +// @Override +// public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { +// builder.startObject() +// .field(_ID, id) +// .field(_VERSION, version); +// +// builder.startObject("tif_config") +// .field(SATIFConfig.FEED_FORMAT_FIELD, feedFormat) +// .field(SATIFConfig.FEED_NAME_FIELD, feedName) +// .field(SATIFConfig.STATE_FIELD, state) +// .field("num_findings", numFindings) +// .field("num_iocs", numIOCs) +// .field(SATIFConfig.ENABLED_TIME_FIELD, lastActivatedTime) +// .field(SATIFConfig.RefreshStats.LAST_REFRESHED_TIME_FIELD, lastRefreshedTime) +// .field("refresh_interval", refreshInterval) +// .field(SATIFConfig.CREATED_BY_USER_FIELD, createdByUser) +// .field(SATIFConfig.FEED_FORMAT_FIELD, feedType) +// .field("license_required", licenseRequired) +// .field("num_scans_configured", numScansConfigured) +// .field("ioc_types", iocTypes) +// .endObject(); +// return builder.endObject(); +// } +// +//} + +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public interface IndexTIFSourceConfigResponse { + String getTIFConfigId(); + Long getVersion(); + TIFSourceConfigDto getTIFConfigDto(); +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java index 847fb3be9..822dcd4d4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java @@ -1,19 +1,21 @@ package org.opensearch.securityanalytics.threatIntel.sacommons; -import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import java.time.Instant; +import java.util.List; import java.util.Map; /** * Threat intel config interface */ public interface TIFSourceConfig { - String getId(); - void setId(String id); + public String getId(); + + public void setId(String id); Long getVersion(); @@ -47,9 +49,9 @@ public interface TIFSourceConfig { void setLastUpdateTime(Instant lastUpdateTime); - Schedule getSchedule(); + IntervalSchedule getSchedule(); - void setSchedule(Schedule schedule); + void setSchedule(IntervalSchedule schedule); TIFJobState getState(); @@ -63,4 +65,8 @@ public interface TIFSourceConfig { void setIocMapStore(Map iocMapStore); + public List getIocTypes(); + + public void setIocTypes(List iocTypes); + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDao.java new file mode 100644 index 000000000..ce925a0f3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDao.java @@ -0,0 +1,11 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.core.action.ActionListener; +public interface TIFSourceConfigDao { + IndexTIFSourceConfigResponse indexTIFConfig(IndexTIFSourceConfigRequest request, ActionListener listener); + + // TODO: + // update + // delete + // get +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java index c8e27d1fa..1899c3af6 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java @@ -1,19 +1,21 @@ package org.opensearch.securityanalytics.threatIntel.sacommons; -import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import java.time.Instant; +import java.util.List; import java.util.Map; /** * Threat intel config dto interface */ public interface TIFSourceConfigDto { - String getId(); - void setId(String id); + public String getId(); + + public void setId(String id); Long getVersion(); @@ -47,9 +49,9 @@ public interface TIFSourceConfigDto { void setLastUpdateTime(Instant lastUpdateTime); - Schedule getSchedule(); + IntervalSchedule getSchedule(); - void setSchedule(Schedule schedule); + void setSchedule(IntervalSchedule schedule); TIFJobState getState(); @@ -62,4 +64,8 @@ public interface TIFSourceConfigDto { Map getIocMapStore(); void setIocMapStore(Map iocMapStore); + + public List getIocTypes(); + + public void setIocTypes(List iocTypes); } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java new file mode 100644 index 000000000..0abc616e9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -0,0 +1,111 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.StepListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +/** + * Service class for threat intel feed source config object + */ +public class SATIFSourceConfigService { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigService.class); + private final SATIFSourceConfigDao satifSourceConfigDao; + private final TIFLockService lockService; + + /** + * Default constructor + * @param satifSourceConfigDao the tif source config dao + * @param lockService the lock service + */ + @Inject + public SATIFSourceConfigService( + final SATIFSourceConfigDao satifSourceConfigDao, + final TIFLockService lockService + ) { + this.satifSourceConfigDao = satifSourceConfigDao; + this.lockService = lockService; + } + + /** + * + * Creates the job index if it doesn't exist and indexes the tif source config object + * + * @param satifSourceConfigDto the tif source config dto + * @param lock the lock object + * @param indexTimeout the index time out + * @param refreshPolicy the refresh policy + * @param listener listener that accepts a tif source config if successful + */ + public void createIndexAndSaveTIFSourceConfig( + final SATIFSourceConfigDto satifSourceConfigDto, + final LockModel lock, + final TimeValue indexTimeout, + WriteRequest.RefreshPolicy refreshPolicy, + final ActionListener listener + ) { + StepListener createIndexStepListener = new StepListener<>(); + createIndexStepListener.whenComplete(v -> { + try { + SATIFSourceConfig satifSourceConfig = convertToSATIFConfig(satifSourceConfigDto); + satifSourceConfig.setState(TIFJobState.AVAILABLE); + satifSourceConfigDao.indexTIFSourceConfig(satifSourceConfig, indexTimeout, refreshPolicy, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig response) { + satifSourceConfig.setId(response.getId()); + satifSourceConfig.setVersion(response.getVersion()); + listener.onResponse(satifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + }, exception -> { + lockService.releaseLock(lock); + log.error("failed to release lock", exception); + listener.onFailure(exception); + }); + satifSourceConfigDao.createJobIndexIfNotExists(createIndexStepListener); + } + + /** + * Converts the DTO to entity + * @param satifSourceConfigDto + * @return satifSourceConfig + */ + public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto satifSourceConfigDto) { + return new SATIFSourceConfig( + satifSourceConfigDto.getId(), + satifSourceConfigDto.getVersion(), + satifSourceConfigDto.getName(), + satifSourceConfigDto.getFeedFormat(), + satifSourceConfigDto.getFeedType(), + satifSourceConfigDto.getCreatedByUser(), + satifSourceConfigDto.getCreatedAt(), + satifSourceConfigDto.getEnabledTime(), + satifSourceConfigDto.getLastUpdateTime(), + satifSourceConfigDto.getSchedule(), + satifSourceConfigDto.getState(), + satifSourceConfigDto.getRefreshType(), + satifSourceConfigDto.getLastRefreshedTime(), + satifSourceConfigDto.getLastRefreshedUser(), + satifSourceConfigDto.isEnabled(), + satifSourceConfigDto.getIocMapStore(), + satifSourceConfigDto.getIocTypes() + ); + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..6b843844c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -0,0 +1,143 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ConcurrentModificationException; + +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Transport action to create threat intel feeds source config object and save IoCs + */ +public class TransportIndexTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportIndexTIFSourceConfigAction.class); + private final SATIFSourceConfigService satifConfigService; + private final TIFLockService lockService; + private final ThreadPool threadPool; + private final Settings settings; + private volatile Boolean filterByEnabled; + private final TimeValue indexTimeout; + + + /** + * Default constructor + * @param transportService the transport service + * @param actionFilters the action filters + * @param threadPool the thread pool + * @param lockService the lock service + */ + @Inject + public TransportIndexTIFSourceConfigAction( + final TransportService transportService, + final ActionFilters actionFilters, + final ThreadPool threadPool, + final SATIFSourceConfigService satifConfigService, + final TIFLockService lockService, + final Settings settings + ) { + super(INDEX_TIF_SOURCE_CONFIG_ACTION_NAME, transportService, actionFilters, SAIndexTIFSourceConfigRequest::new); + this.threadPool = threadPool; + this.satifConfigService = satifConfigService; + this.lockService = lockService; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); + } + + + @Override + protected void doExecute(final Task task, final SAIndexTIFSourceConfigRequest request, final ActionListener listener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + + retrieveLockAndCreateTIFConfig(request, listener, user); + } + + private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest request, ActionListener listener, User user) { + try { + lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + listener.onFailure( + new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") + ); + log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST); + return; + } + try { + SATIFSourceConfigDto satifConfigDto = request.getTIFConfigDto(); + if (user != null) { + satifConfigDto.setCreatedByUser(user.getName()); + } + try { + satifConfigService.createIndexAndSaveTIFSourceConfig(satifConfigDto, + lock, + indexTimeout, + request.getRefreshPolicy(), + new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig satifSourceConfig) { + SATIFSourceConfigDto satifSourceConfigDto = new SATIFSourceConfigDto(satifSourceConfig); + listener.onResponse(new SAIndexTIFSourceConfigResponse(satifSourceConfigDto.getId(), satifSourceConfigDto.getVersion(), RestStatus.OK, satifSourceConfigDto)); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + log.error("listener failed when executing", e); + } + + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + log.error("listener failed when executing", e); + } + }, exception -> { + listener.onFailure(exception); + log.error("execution failed", exception); + })); + } catch (Exception e) { + log.error("Failed to acquire lock for job", e); + listener.onFailure(e); + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index c04c08798..2c756b3d3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -123,7 +123,6 @@ protected void internalDoExecute( listener.onFailure(exception); }); tifJobParameterService.createJobIndexIfNotExists(createIndexStepListener); - } /** diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index 59d49f73d..ee258a11d 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,12 +1,14 @@ { - "dynamic": "strict", "_meta" : { "schema_version": 2 }, + "dynamic": "strict", "properties": { - "feed_format_config": { - "dynamic": "false", + "feed_source_config": { "properties": { + "version": { + "type": "long" + }, "feed_name": { "type" : "text", "fields" : { @@ -92,8 +94,16 @@ "enabled": { "type": "boolean" }, - "version": { - "type": "long" + "ioc_map_store": { + "type": "object" + }, + "ioc_types": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } } } }, diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 0b5880bad..53654f4ae 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -67,6 +67,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -662,6 +663,9 @@ protected HttpEntity toHttpEntity(UpdateIndexMappingsRequest request) throws IOE protected HttpEntity toHttpEntity(CorrelationRule rule) throws IOException { return new StringEntity(toJsonString(rule), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(SATIFSourceConfigDto satifSourceConfigDto) throws IOException { + return new StringEntity(toJsonString(satifSourceConfigDto), ContentType.APPLICATION_JSON); + } protected RestStatus restStatus(Response response) { return RestStatus.fromCode(response.getStatusLine().getStatusCode()); @@ -706,6 +710,11 @@ protected String toJsonString(ThreatIntelFeedData tifd) throws IOException { return IndexUtilsKt.string(shuffleXContent(tifd.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(SATIFSourceConfigDto satifSourceConfigDto) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(satifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String alertingScheduledJobMappings() { return " \"_meta\" : {\n" + " \"schema_version\": 5\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java new file mode 100644 index 000000000..32fe12a51 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; + +public class SATIFSourceConfigRestApiIT extends SecurityAnalyticsRestTestCase { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigRestApiIT.class); + public void testCreateSATIFSourceConfig() throws IOException { + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + + SATIFSourceConfigDto satifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "feedname", + "stix", + FeedType.CUSTOM, + null, + null, + null, + null, + schedule, + null, + null, + null, + null, + true, + null, + List.of("ip", "dns") + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.TIF_CONFIG_URI, Collections.emptyMap(), toHttpEntity(satifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_CONFIG_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + } +} From 8be3664bd142e398a17805ac820b3dda89961fc2 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 23 May 2024 17:56:11 -0700 Subject: [PATCH 2/7] clean up Signed-off-by: Joanne Wang --- .../SAIndexTIFSourceConfigResponse.java | 123 ------------------ .../IndexTIFSourceConfigResponse.java | 123 ------------------ 2 files changed, 246 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java index 457da54ef..5904c13ab 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -1,126 +1,3 @@ -///* -// * Copyright OpenSearch Contributors -// * SPDX-License-Identifier: Apache-2.0 -// */ -//package org.opensearch.securityanalytics.threatIntel.action; -// -//import org.opensearch.core.action.ActionResponse; -//import org.opensearch.core.common.io.stream.StreamInput; -//import org.opensearch.core.common.io.stream.StreamOutput; -//import org.opensearch.core.xcontent.ToXContentObject; -//import org.opensearch.core.xcontent.XContentBuilder; -//import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -//import org.opensearch.securityanalytics.threatIntel.model.SATIFConfig; -// -//import java.io.IOException; -//import java.time.Instant; -//import java.util.List; -// -//import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; -//import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; -// -//public class IndexTIFConfigResponse extends ActionResponse implements ToXContentObject { -// private String id; -// private Long version; -// private String feedFormat; -// private String feedName; -// private TIFJobState state; -// private Integer numFindings; -// private Integer numIOCs; -// private Instant lastActivatedTime; -// private Instant lastRefreshedTime; -// private String refreshInterval; -// -//// private Source source; -// private String createdByUser; -// private String feedType; -// private Boolean licenseRequired; -// private Integer numScansConfigured; -// private List iocTypes; -// -// public IndexTIFConfigResponse(String feedId, Long version, String feedFormat, String feedName, TIFJobState state, Integer numFindings, -// Integer numIOCs, Instant lastActivatedTime, Instant lastRefreshedTime, String refreshInterval, String createdByUser, -// String feedType, Boolean licenseRequired, Integer numScansConfigured, List iocTypes) { -// super(); -// this.id = feedId; -// this.version = version; -// this.feedFormat = feedFormat; -// this.feedName = feedName; -// this.state = state; -// this.numFindings = numFindings; -// this.numIOCs = numIOCs; -// this.lastActivatedTime = lastActivatedTime; -// this.lastRefreshedTime = lastRefreshedTime; -// this.refreshInterval = refreshInterval; -// this.createdByUser = createdByUser; -// this.feedType = feedType; -// this.licenseRequired = licenseRequired; -// this.numScansConfigured = numScansConfigured; -// this.iocTypes = iocTypes; -// } -// -// public IndexTIFConfigResponse(StreamInput sin) throws IOException { -// this.id = sin.readString(); -// this.version = sin.readLong(); -// this.feedFormat = sin.readString(); -// this.feedName = sin.readString(); -// this.state = TIFJobState.valueOf(sin.readString()); -// this.numFindings = sin.readInt(); -// this.numIOCs = sin.readInt(); -// this.lastActivatedTime = sin.readInstant(); -// this.lastRefreshedTime = sin.readInstant(); -// this.refreshInterval = sin.readString(); -// this.createdByUser = sin.readString(); -// this.feedType = sin.readString(); -// this.licenseRequired = sin.readBoolean(); -// this.numScansConfigured = sin.readInt(); -// this.iocTypes = sin.readStringList(); -// } -// -// @Override -// public void writeTo(StreamOutput out) throws IOException { -// out.writeString(id); -// out.writeString(feedFormat); -// out.writeString(feedName); -// out.writeString(state.name()); -// out.writeInt(numFindings); -// out.writeInt(numIOCs); -// out.writeInstant(lastActivatedTime); -// out.writeInstant(lastRefreshedTime); -// out.writeString(refreshInterval); -// out.writeString(createdByUser); -// out.writeString(feedType); -// out.writeBoolean(licenseRequired); -// out.writeInt(numScansConfigured); -// out.writeStringCollection(iocTypes); -// } -// -// @Override -// public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { -// builder.startObject() -// .field(_ID, id) -// .field(_VERSION, version); -// -// builder.startObject("tif_config") -// .field(SATIFConfig.FEED_FORMAT_FIELD, feedFormat) -// .field(SATIFConfig.FEED_NAME_FIELD, feedName) -// .field(SATIFConfig.STATE_FIELD, state) -// .field("num_findings", numFindings) -// .field("num_iocs", numIOCs) -// .field(SATIFConfig.ENABLED_TIME_FIELD, lastActivatedTime) -// .field(SATIFConfig.RefreshStats.LAST_REFRESHED_TIME_FIELD, lastRefreshedTime) -// .field("refresh_interval", refreshInterval) -// .field(SATIFConfig.CREATED_BY_USER_FIELD, createdByUser) -// .field(SATIFConfig.FEED_FORMAT_FIELD, feedType) -// .field("license_required", licenseRequired) -// .field("num_scans_configured", numScansConfigured) -// .field("ioc_types", iocTypes) -// .endObject(); -// return builder.endObject(); -// } -// -//} - /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java index 8cbc1b025..5a9e4daa6 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java @@ -1,126 +1,3 @@ -///* -// * Copyright OpenSearch Contributors -// * SPDX-License-Identifier: Apache-2.0 -// */ -//package org.opensearch.securityanalytics.threatIntel.action; -// -//import org.opensearch.core.action.ActionResponse; -//import org.opensearch.core.common.io.stream.StreamInput; -//import org.opensearch.core.common.io.stream.StreamOutput; -//import org.opensearch.core.xcontent.ToXContentObject; -//import org.opensearch.core.xcontent.XContentBuilder; -//import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -//import org.opensearch.securityanalytics.threatIntel.model.SATIFConfig; -// -//import java.io.IOException; -//import java.time.Instant; -//import java.util.List; -// -//import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; -//import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; -// -//public class IndexTIFConfigResponse extends ActionResponse implements ToXContentObject { -// private String id; -// private Long version; -// private String feedFormat; -// private String feedName; -// private TIFJobState state; -// private Integer numFindings; -// private Integer numIOCs; -// private Instant lastActivatedTime; -// private Instant lastRefreshedTime; -// private String refreshInterval; -// -//// private Source source; -// private String createdByUser; -// private String feedType; -// private Boolean licenseRequired; -// private Integer numScansConfigured; -// private List iocTypes; -// -// public IndexTIFConfigResponse(String feedId, Long version, String feedFormat, String feedName, TIFJobState state, Integer numFindings, -// Integer numIOCs, Instant lastActivatedTime, Instant lastRefreshedTime, String refreshInterval, String createdByUser, -// String feedType, Boolean licenseRequired, Integer numScansConfigured, List iocTypes) { -// super(); -// this.id = feedId; -// this.version = version; -// this.feedFormat = feedFormat; -// this.feedName = feedName; -// this.state = state; -// this.numFindings = numFindings; -// this.numIOCs = numIOCs; -// this.lastActivatedTime = lastActivatedTime; -// this.lastRefreshedTime = lastRefreshedTime; -// this.refreshInterval = refreshInterval; -// this.createdByUser = createdByUser; -// this.feedType = feedType; -// this.licenseRequired = licenseRequired; -// this.numScansConfigured = numScansConfigured; -// this.iocTypes = iocTypes; -// } -// -// public IndexTIFConfigResponse(StreamInput sin) throws IOException { -// this.id = sin.readString(); -// this.version = sin.readLong(); -// this.feedFormat = sin.readString(); -// this.feedName = sin.readString(); -// this.state = TIFJobState.valueOf(sin.readString()); -// this.numFindings = sin.readInt(); -// this.numIOCs = sin.readInt(); -// this.lastActivatedTime = sin.readInstant(); -// this.lastRefreshedTime = sin.readInstant(); -// this.refreshInterval = sin.readString(); -// this.createdByUser = sin.readString(); -// this.feedType = sin.readString(); -// this.licenseRequired = sin.readBoolean(); -// this.numScansConfigured = sin.readInt(); -// this.iocTypes = sin.readStringList(); -// } -// -// @Override -// public void writeTo(StreamOutput out) throws IOException { -// out.writeString(id); -// out.writeString(feedFormat); -// out.writeString(feedName); -// out.writeString(state.name()); -// out.writeInt(numFindings); -// out.writeInt(numIOCs); -// out.writeInstant(lastActivatedTime); -// out.writeInstant(lastRefreshedTime); -// out.writeString(refreshInterval); -// out.writeString(createdByUser); -// out.writeString(feedType); -// out.writeBoolean(licenseRequired); -// out.writeInt(numScansConfigured); -// out.writeStringCollection(iocTypes); -// } -// -// @Override -// public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { -// builder.startObject() -// .field(_ID, id) -// .field(_VERSION, version); -// -// builder.startObject("tif_config") -// .field(SATIFConfig.FEED_FORMAT_FIELD, feedFormat) -// .field(SATIFConfig.FEED_NAME_FIELD, feedName) -// .field(SATIFConfig.STATE_FIELD, state) -// .field("num_findings", numFindings) -// .field("num_iocs", numIOCs) -// .field(SATIFConfig.ENABLED_TIME_FIELD, lastActivatedTime) -// .field(SATIFConfig.RefreshStats.LAST_REFRESHED_TIME_FIELD, lastRefreshedTime) -// .field("refresh_interval", refreshInterval) -// .field(SATIFConfig.CREATED_BY_USER_FIELD, createdByUser) -// .field(SATIFConfig.FEED_FORMAT_FIELD, feedType) -// .field("license_required", licenseRequired) -// .field("num_scans_configured", numScansConfigured) -// .field("ioc_types", iocTypes) -// .endObject(); -// return builder.endObject(); -// } -// -//} - /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 From 491bbaf44c1851bac42f5f3f79abb01541e4d82f Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Fri, 24 May 2024 09:24:56 -0700 Subject: [PATCH 3/7] tif/source Signed-off-by: Joanne Wang --- .../securityanalytics/SecurityAnalyticsPlugin.java | 3 ++- .../resthandler/RestIndexTIFConfigAction.java | 10 ++++------ .../resthandler/SATIFSourceConfigRestApiIT.java | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index de28cbfbc..3307c7acc 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -111,7 +111,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; - public static final String TIF_CONFIG_URI = PLUGINS_BASE_URI + "/tif"; + public static final String TIF_BASE_URI = PLUGINS_BASE_URI + "/tif"; + public static final String TIF_SOURCE_CONFIG_URI = PLUGINS_BASE_URI + TIF_BASE_URI + "/source"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; public static final String JOB_TYPE = "opensearch_sap_job"; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java index ff6243cef..017f2232a 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java @@ -30,8 +30,6 @@ import java.util.List; import java.util.Locale; -import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.NO_ID; - public class RestIndexTIFConfigAction extends BaseRestHandler { private static final Logger log = LogManager.getLogger(RestIndexTIFConfigAction.class); @@ -42,14 +40,14 @@ public String getName() { @Override public List routes() { return List.of( - new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.TIF_CONFIG_URI), - new Route(RestRequest.Method.PUT, SecurityAnalyticsPlugin.TIF_CONFIG_URI + "/{tifConfigId}") + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI), + new Route(RestRequest.Method.PUT, SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI + "/{tifConfigId}") ); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.TIF_CONFIG_URI)); + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI)); WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE; if (request.hasParam(RestHandlerUtils.REFRESH)) { @@ -80,7 +78,7 @@ public RestResponse buildResponse(SAIndexTIFSourceConfigResponse response) throw BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); if (restMethod == RestRequest.Method.POST) { - String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_CONFIG_URI, response.getTIFConfigId()); + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI, response.getTIFConfigId()); restResponse.addHeader("Location", location); } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java index 32fe12a51..95cbadf0e 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -53,7 +53,7 @@ public void testCreateSATIFSourceConfig() throws IOException { List.of("ip", "dns") ); - Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.TIF_CONFIG_URI, Collections.emptyMap(), toHttpEntity(satifSourceConfigDto)); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI, Collections.emptyMap(), toHttpEntity(satifSourceConfigDto)); Assert.assertEquals(201, response.getStatusLine().getStatusCode()); Map responseBody = asMap(response); @@ -62,7 +62,7 @@ public void testCreateSATIFSourceConfig() throws IOException { int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); Assert.assertTrue("incorrect version", createdVersion > 0); - Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_CONFIG_URI, createdId), response.getHeader("Location")); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI, createdId), response.getHeader("Location")); String request = "{\n" + " \"query\" : {\n" + From f850464a7f2d39b650cb7e5d851ef90e0a65bccc Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Fri, 24 May 2024 15:13:18 -0700 Subject: [PATCH 4/7] fix uri Signed-off-by: Joanne Wang --- .../opensearch/securityanalytics/SecurityAnalyticsPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 3307c7acc..6de8eecc0 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -112,7 +112,7 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; public static final String TIF_BASE_URI = PLUGINS_BASE_URI + "/tif"; - public static final String TIF_SOURCE_CONFIG_URI = PLUGINS_BASE_URI + TIF_BASE_URI + "/source"; + public static final String TIF_SOURCE_CONFIG_URI = PLUGINS_BASE_URI + "/tif/source"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; public static final String JOB_TYPE = "opensearch_sap_job"; From f41ac1670751c81dc6312749f5b90c390b1e3612 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Fri, 24 May 2024 22:20:04 -0700 Subject: [PATCH 5/7] comments Signed-off-by: Joanne Wang --- .../SecurityAnalyticsPlugin.java | 18 ++--- .../action/SAIndexTIFSourceConfigRequest.java | 21 +++--- .../SAIndexTIFSourceConfigResponse.java | 40 +++++------ .../threatIntel/dao/SATIFSourceConfigDao.java | 41 ++++++----- .../model/SATIFSourceConfigDto.java | 36 +++++----- ...va => RestIndexTIFSourceConfigAction.java} | 12 ++-- .../service/SATIFSourceConfigService.java | 68 +++++++++---------- .../TransportIndexTIFSourceConfigAction.java | 19 +++--- .../SecurityAnalyticsRestTestCase.java | 8 +-- .../SATIFSourceConfigRestApiIT.java | 6 +- 10 files changed, 129 insertions(+), 140 deletions(-) rename src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/{RestIndexTIFConfigAction.java => RestIndexTIFSourceConfigAction.java} (91%) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 6de8eecc0..6cb8138fd 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -68,7 +68,7 @@ import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; -import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; @@ -111,8 +111,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; - public static final String TIF_BASE_URI = PLUGINS_BASE_URI + "/tif"; - public static final String TIF_SOURCE_CONFIG_URI = PLUGINS_BASE_URI + "/tif/source"; + public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel"; + public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; public static final String JOB_TYPE = "opensearch_sap_job"; @@ -141,7 +141,7 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private LogTypeService logTypeService; - private SATIFSourceConfigDao satifSourceConfigDao; + private SATIFSourceConfigDao SaTifSourceConfigDao; @Override public Collection getSystemIndexDescriptors(Settings settings){ @@ -179,8 +179,8 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); - satifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool); - SATIFSourceConfigService satifSourceConfigService = new SATIFSourceConfigService(satifSourceConfigDao, threatIntelLockService); + SaTifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool); + SATIFSourceConfigService SaTifSourceConfigService = new SATIFSourceConfigService(SaTifSourceConfigDao, threatIntelLockService); TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); @@ -188,7 +188,7 @@ public Collection createComponents(Client client, return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService, satifSourceConfigDao, satifSourceConfigService); + tifJobUpdateService, tifJobParameterService, threatIntelLockService, SaTifSourceConfigDao, SaTifSourceConfigService); } @Override @@ -229,7 +229,7 @@ public List getRestHandlers(Settings settings, new RestIndexCustomLogTypeAction(), new RestSearchCustomLogTypeAction(), new RestDeleteCustomLogTypeAction(), - new RestIndexTIFConfigAction() + new RestIndexTIFSourceConfigAction() ); } @@ -259,7 +259,7 @@ public ScheduledJobParser getJobParser() { case FEED_SOURCE_CONFIG_FIELD: return SATIFSourceConfig.parse(xcp, id, null); default: - log.warn("Unsupported document was indexed"); + log.error("Job parser failed for [{}] in security analytics job registration", fieldName); xcp.skipChildren(); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java index 073b2a4b8..a9c73d63f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java @@ -8,19 +8,14 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.WriteRequest; -import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.rest.RestRequest; import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigRequest; import java.io.IOException; -import java.sql.Time; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.List; /** @@ -31,17 +26,17 @@ public class SAIndexTIFSourceConfigRequest extends ActionRequest implements Inde private String tifSourceConfigId; private final WriteRequest.RefreshPolicy refreshPolicy; private final RestRequest.Method method; - private SATIFSourceConfigDto satifSourceConfigDto; + private SATIFSourceConfigDto SaTifSourceConfigDto; public SAIndexTIFSourceConfigRequest(String tifSourceConfigId, WriteRequest.RefreshPolicy refreshPolicy, RestRequest.Method method, - SATIFSourceConfigDto satifSourceConfigDto) { + SATIFSourceConfigDto SaTifSourceConfigDto) { super(); this.tifSourceConfigId = tifSourceConfigId; this.refreshPolicy = refreshPolicy; this.method = method; - this.satifSourceConfigDto = satifSourceConfigDto; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; } public SAIndexTIFSourceConfigRequest(StreamInput sin) throws IOException { @@ -58,7 +53,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(tifSourceConfigId); refreshPolicy.writeTo(out); out.writeEnum(method); - satifSourceConfigDto.writeTo(out); + SaTifSourceConfigDto.writeTo(out); } @Override @@ -72,11 +67,11 @@ public void setTIFConfigId(String tifConfigId) { @Override public SATIFSourceConfigDto getTIFConfigDto() { - return satifSourceConfigDto; + return SaTifSourceConfigDto; } - public void setTIFConfigDto(SATIFSourceConfigDto saTifConfigDto) { - this.satifSourceConfigDto = saTifConfigDto; + public void setTIFConfigDto(SATIFSourceConfigDto SaTifSourceConfigDto) { + this.SaTifSourceConfigDto = SaTifSourceConfigDto; } public WriteRequest.RefreshPolicy getRefreshPolicy() { @@ -86,7 +81,7 @@ public WriteRequest.RefreshPolicy getRefreshPolicy() { @Override public ActionRequestValidationException validate() { ActionRequestValidationException errors = new ActionRequestValidationException(); - List errorMsgs = VALIDATOR.validateTIFJobName(satifSourceConfigDto.getName()); + List errorMsgs = VALIDATOR.validateTIFJobName(SaTifSourceConfigDto.getName()); if (errorMsgs.isEmpty() == false) { errorMsgs.forEach(errors::addValidationError); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java index 5904c13ab..b4ea1b9c0 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -20,17 +20,17 @@ import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; public class SAIndexTIFSourceConfigResponse extends ActionResponse implements ToXContentObject, IndexTIFSourceConfigResponse { - private final String tifConfigId; + private final String id; private final Long version; private final RestStatus status; - private final SATIFSourceConfigDto saTIFConfigDto; + private final SATIFSourceConfigDto SaTifSourceConfigDto; - public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto tifConfig) { + public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto SaTifSourceConfigDto) { super(); - this.tifConfigId = id; + this.id = id; this.version = version; this.status = status; - this.saTIFConfigDto = tifConfig; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; } public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException { @@ -44,37 +44,37 @@ public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(tifConfigId); + out.writeString(id); out.writeLong(version); out.writeEnum(status); - saTIFConfigDto.writeTo(out); + SaTifSourceConfigDto.writeTo(out); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject() - .field(_ID, tifConfigId) + .field(_ID, id) .field(_VERSION, version); builder.startObject("tif_config") - .field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, saTIFConfigDto.getFeedFormat()) - .field(SATIFSourceConfigDto.FEED_NAME_FIELD, saTIFConfigDto.getName()) - .field(SATIFSourceConfigDto.FEED_TYPE_FIELD, saTIFConfigDto.getFeedType()) - .field(SATIFSourceConfigDto.STATE_FIELD, saTIFConfigDto.getState()) - .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTIFConfigDto.getEnabledTime()) - .field(SATIFSourceConfigDto.ENABLED_FIELD, saTIFConfigDto.isEnabled()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTIFConfigDto.getLastRefreshedTime()) - .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTIFConfigDto.getSchedule()) + .field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, SaTifSourceConfigDto.getFeedFormat()) + .field(SATIFSourceConfigDto.FEED_NAME_FIELD, SaTifSourceConfigDto.getName()) + .field(SATIFSourceConfigDto.FEED_TYPE_FIELD, SaTifSourceConfigDto.getFeedType()) + .field(SATIFSourceConfigDto.STATE_FIELD, SaTifSourceConfigDto.getState()) + .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, SaTifSourceConfigDto.getEnabledTime()) + .field(SATIFSourceConfigDto.ENABLED_FIELD, SaTifSourceConfigDto.isEnabled()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, SaTifSourceConfigDto.getLastRefreshedTime()) + .field(SATIFSourceConfigDto.SCHEDULE_FIELD, SaTifSourceConfigDto.getSchedule()) // source - .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTIFConfigDto.getCreatedByUser()) - .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTIFConfigDto.getIocTypes()) + .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, SaTifSourceConfigDto.getCreatedByUser()) + .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, SaTifSourceConfigDto.getIocTypes()) .endObject(); return builder.endObject(); } @Override public String getTIFConfigId() { - return tifConfigId; + return id; } @Override public Long getVersion() { @@ -82,7 +82,7 @@ public Long getVersion() { } @Override public TIFSourceConfigDto getTIFConfigDto() { - return saTIFConfigDto; + return SaTifSourceConfigDto; } public RestStatus getStatus() { return status; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java index 5078bc85b..67d846d90 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java @@ -57,27 +57,24 @@ public SATIFSourceConfigDao(final Client client, final ClusterService clusterSer this.threadPool = threadPool; } - public void indexTIFSourceConfig(SATIFSourceConfig satifSourceConfig, + public void indexTIFSourceConfig(SATIFSourceConfig SaTifSourceConfigDto, TimeValue indexTimeout, - WriteRequest.RefreshPolicy refreshPolicy, - final ActionListener actionListener) throws Exception { - IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) - .setRefreshPolicy(refreshPolicy) - .source(satifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .timeout(indexTimeout); - log.debug("Indexing tif source config"); - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(IndexResponse response) { - log.debug("TIF source config indexed success."); - satifSourceConfig.setId(response.getId()); - actionListener.onResponse(satifSourceConfig); - } - @Override - public void onFailure(Exception e) { - throw new SecurityAnalyticsException("Exception saving the tif source config in index", RestStatus.INTERNAL_SERVER_ERROR, e); - } - }); + final ActionListener actionListener) { + try { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(SaTifSourceConfigDto.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + log.debug("Indexing tif source config"); + client.index(indexRequest, ActionListener.wrap(response -> { + log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); + SaTifSourceConfigDto.setId(response.getId()); + actionListener.onResponse(SaTifSourceConfigDto); + }, actionListener::onFailure)); + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } } public ThreadPool getThreadPool() { @@ -94,8 +91,8 @@ private String getIndexMapping() { } } } catch (IOException e) { - log.error("Runtime exception when getting the threat intel index mapping", e); - throw new SecurityAnalyticsException("Runtime exception when getting the threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + log.error("Failed to get the threat intel index mapping", e); + throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index c5eb1c6a8..dfba113ee 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -81,24 +81,24 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou private Map iocMapStore; private List iocTypes; - public SATIFSourceConfigDto(SATIFSourceConfig saTIFSourceConfig) { - this.id = saTIFSourceConfig.getId(); - this.version = saTIFSourceConfig.getVersion(); - this.feedName = saTIFSourceConfig.getName(); - this.feedFormat = saTIFSourceConfig.getFeedFormat(); - this.feedType = saTIFSourceConfig.getFeedType(); - this.createdByUser = saTIFSourceConfig.getCreatedByUser(); - this.createdAt = saTIFSourceConfig.getCreatedAt(); - this.enabledTime = saTIFSourceConfig.getEnabledTime(); - this.lastUpdateTime = saTIFSourceConfig.getLastUpdateTime(); - this.schedule = saTIFSourceConfig.getSchedule(); - this.state = saTIFSourceConfig.getState();; - this.refreshType = saTIFSourceConfig.getRefreshType(); - this.lastRefreshedTime = saTIFSourceConfig.getLastRefreshedTime(); - this.lastRefreshedUser = saTIFSourceConfig.getLastRefreshedUser(); - this.isEnabled = saTIFSourceConfig.isEnabled();; - this.iocMapStore = saTIFSourceConfig.getIocMapStore(); - this.iocTypes = saTIFSourceConfig.getIocTypes(); + public SATIFSourceConfigDto(SATIFSourceConfig SaTifSourceConfig) { + this.id = SaTifSourceConfig.getId(); + this.version = SaTifSourceConfig.getVersion(); + this.feedName = SaTifSourceConfig.getName(); + this.feedFormat = SaTifSourceConfig.getFeedFormat(); + this.feedType = SaTifSourceConfig.getFeedType(); + this.createdByUser = SaTifSourceConfig.getCreatedByUser(); + this.createdAt = SaTifSourceConfig.getCreatedAt(); + this.enabledTime = SaTifSourceConfig.getEnabledTime(); + this.lastUpdateTime = SaTifSourceConfig.getLastUpdateTime(); + this.schedule = SaTifSourceConfig.getSchedule(); + this.state = SaTifSourceConfig.getState();; + this.refreshType = SaTifSourceConfig.getRefreshType(); + this.lastRefreshedTime = SaTifSourceConfig.getLastRefreshedTime(); + this.lastRefreshedUser = SaTifSourceConfig.getLastRefreshedUser(); + this.isEnabled = SaTifSourceConfig.isEnabled();; + this.iocMapStore = SaTifSourceConfig.getIocMapStore(); + this.iocTypes = SaTifSourceConfig.getIocTypes(); } public SATIFSourceConfigDto(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java similarity index 91% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java index 017f2232a..0545048c4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java @@ -31,8 +31,8 @@ import java.util.Locale; -public class RestIndexTIFConfigAction extends BaseRestHandler { - private static final Logger log = LogManager.getLogger(RestIndexTIFConfigAction.class); +public class RestIndexTIFSourceConfigAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestIndexTIFSourceConfigAction.class); @Override public String getName() { return "index_tif_config_action"; @@ -40,14 +40,14 @@ public String getName() { @Override public List routes() { return List.of( - new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI), - new Route(RestRequest.Method.PUT, SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI + "/{tifConfigId}") + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI), + new Route(RestRequest.Method.PUT, SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/{tifConfigId}") ); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI)); + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI)); WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE; if (request.hasParam(RestHandlerUtils.REFRESH)) { @@ -78,7 +78,7 @@ public RestResponse buildResponse(SAIndexTIFSourceConfigResponse response) throw BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); if (restMethod == RestRequest.Method.POST) { - String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI, response.getTIFConfigId()); + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, response.getTIFConfigId()); restResponse.addHeader("Location", location); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 0abc616e9..740440fb5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -19,20 +19,20 @@ */ public class SATIFSourceConfigService { private static final Logger log = LogManager.getLogger(SATIFSourceConfigService.class); - private final SATIFSourceConfigDao satifSourceConfigDao; + private final SATIFSourceConfigDao SaTifSourceConfigDao; private final TIFLockService lockService; /** * Default constructor - * @param satifSourceConfigDao the tif source config dao + * @param SaTifSourceConfigDao the tif source config dao * @param lockService the lock service */ @Inject public SATIFSourceConfigService( - final SATIFSourceConfigDao satifSourceConfigDao, + final SATIFSourceConfigDao SaTifSourceConfigDao, final TIFLockService lockService ) { - this.satifSourceConfigDao = satifSourceConfigDao; + this.SaTifSourceConfigDao = SaTifSourceConfigDao; this.lockService = lockService; } @@ -40,30 +40,28 @@ public SATIFSourceConfigService( * * Creates the job index if it doesn't exist and indexes the tif source config object * - * @param satifSourceConfigDto the tif source config dto + * @param SaTifSourceConfigDto the tif source config dto * @param lock the lock object * @param indexTimeout the index time out - * @param refreshPolicy the refresh policy * @param listener listener that accepts a tif source config if successful */ public void createIndexAndSaveTIFSourceConfig( - final SATIFSourceConfigDto satifSourceConfigDto, + final SATIFSourceConfigDto SaTifSourceConfigDto, final LockModel lock, final TimeValue indexTimeout, - WriteRequest.RefreshPolicy refreshPolicy, final ActionListener listener ) { StepListener createIndexStepListener = new StepListener<>(); createIndexStepListener.whenComplete(v -> { try { - SATIFSourceConfig satifSourceConfig = convertToSATIFConfig(satifSourceConfigDto); - satifSourceConfig.setState(TIFJobState.AVAILABLE); - satifSourceConfigDao.indexTIFSourceConfig(satifSourceConfig, indexTimeout, refreshPolicy, new ActionListener<>() { + SATIFSourceConfig SaTifSourceConfig = convertToSATIFConfig(SaTifSourceConfigDto); + SaTifSourceConfig.setState(TIFJobState.AVAILABLE); + SaTifSourceConfigDao.indexTIFSourceConfig(SaTifSourceConfig, indexTimeout, new ActionListener<>() { @Override public void onResponse(SATIFSourceConfig response) { - satifSourceConfig.setId(response.getId()); - satifSourceConfig.setVersion(response.getVersion()); - listener.onResponse(satifSourceConfig); + SaTifSourceConfig.setId(response.getId()); + SaTifSourceConfig.setVersion(response.getVersion()); + listener.onResponse(SaTifSourceConfig); } @Override public void onFailure(Exception e) { @@ -78,33 +76,33 @@ public void onFailure(Exception e) { log.error("failed to release lock", exception); listener.onFailure(exception); }); - satifSourceConfigDao.createJobIndexIfNotExists(createIndexStepListener); + SaTifSourceConfigDao.createJobIndexIfNotExists(createIndexStepListener); } /** * Converts the DTO to entity - * @param satifSourceConfigDto - * @return satifSourceConfig + * @param SaTifSourceConfigDto + * @return SaTifSourceConfig */ - public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto satifSourceConfigDto) { + public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto SaTifSourceConfigDto) { return new SATIFSourceConfig( - satifSourceConfigDto.getId(), - satifSourceConfigDto.getVersion(), - satifSourceConfigDto.getName(), - satifSourceConfigDto.getFeedFormat(), - satifSourceConfigDto.getFeedType(), - satifSourceConfigDto.getCreatedByUser(), - satifSourceConfigDto.getCreatedAt(), - satifSourceConfigDto.getEnabledTime(), - satifSourceConfigDto.getLastUpdateTime(), - satifSourceConfigDto.getSchedule(), - satifSourceConfigDto.getState(), - satifSourceConfigDto.getRefreshType(), - satifSourceConfigDto.getLastRefreshedTime(), - satifSourceConfigDto.getLastRefreshedUser(), - satifSourceConfigDto.isEnabled(), - satifSourceConfigDto.getIocMapStore(), - satifSourceConfigDto.getIocTypes() + SaTifSourceConfigDto.getId(), + SaTifSourceConfigDto.getVersion(), + SaTifSourceConfigDto.getName(), + SaTifSourceConfigDto.getFeedFormat(), + SaTifSourceConfigDto.getFeedType(), + SaTifSourceConfigDto.getCreatedByUser(), + SaTifSourceConfigDto.getCreatedAt(), + SaTifSourceConfigDto.getEnabledTime(), + SaTifSourceConfigDto.getLastUpdateTime(), + SaTifSourceConfigDto.getSchedule(), + SaTifSourceConfigDto.getState(), + SaTifSourceConfigDto.getRefreshType(), + SaTifSourceConfigDto.getLastRefreshedTime(), + SaTifSourceConfigDto.getLastRefreshedUser(), + SaTifSourceConfigDto.isEnabled(), + SaTifSourceConfigDto.getIocMapStore(), + SaTifSourceConfigDto.getIocTypes() ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java index 6b843844c..c64341521 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -39,7 +39,7 @@ */ public class TransportIndexTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { private static final Logger log = LogManager.getLogger(TransportIndexTIFSourceConfigAction.class); - private final SATIFSourceConfigService satifConfigService; + private final SATIFSourceConfigService SaTifSourceConfigService; private final TIFLockService lockService; private final ThreadPool threadPool; private final Settings settings; @@ -59,13 +59,13 @@ public TransportIndexTIFSourceConfigAction( final TransportService transportService, final ActionFilters actionFilters, final ThreadPool threadPool, - final SATIFSourceConfigService satifConfigService, + final SATIFSourceConfigService SaTifSourceConfigService, final TIFLockService lockService, final Settings settings ) { super(INDEX_TIF_SOURCE_CONFIG_ACTION_NAME, transportService, actionFilters, SAIndexTIFSourceConfigRequest::new); this.threadPool = threadPool; - this.satifConfigService = satifConfigService; + this.SaTifSourceConfigService = SaTifSourceConfigService; this.lockService = lockService; this.settings = settings; this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); @@ -98,20 +98,19 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques return; } try { - SATIFSourceConfigDto satifConfigDto = request.getTIFConfigDto(); + SATIFSourceConfigDto SaTifSourceConfigDto = request.getTIFConfigDto(); if (user != null) { - satifConfigDto.setCreatedByUser(user.getName()); + SaTifSourceConfigDto.setCreatedByUser(user.getName()); } try { - satifConfigService.createIndexAndSaveTIFSourceConfig(satifConfigDto, + SaTifSourceConfigService.createIndexAndSaveTIFSourceConfig(SaTifSourceConfigDto, lock, indexTimeout, - request.getRefreshPolicy(), new ActionListener<>() { @Override - public void onResponse(SATIFSourceConfig satifSourceConfig) { - SATIFSourceConfigDto satifSourceConfigDto = new SATIFSourceConfigDto(satifSourceConfig); - listener.onResponse(new SAIndexTIFSourceConfigResponse(satifSourceConfigDto.getId(), satifSourceConfigDto.getVersion(), RestStatus.OK, satifSourceConfigDto)); + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto(SaTifSourceConfig); + listener.onResponse(new SAIndexTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto)); } @Override public void onFailure(Exception e) { diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 53654f4ae..91289a91e 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -663,8 +663,8 @@ protected HttpEntity toHttpEntity(UpdateIndexMappingsRequest request) throws IOE protected HttpEntity toHttpEntity(CorrelationRule rule) throws IOException { return new StringEntity(toJsonString(rule), ContentType.APPLICATION_JSON); } - protected HttpEntity toHttpEntity(SATIFSourceConfigDto satifSourceConfigDto) throws IOException { - return new StringEntity(toJsonString(satifSourceConfigDto), ContentType.APPLICATION_JSON); + protected HttpEntity toHttpEntity(SATIFSourceConfigDto SaTifSourceConfigDto) throws IOException { + return new StringEntity(toJsonString(SaTifSourceConfigDto), ContentType.APPLICATION_JSON); } protected RestStatus restStatus(Response response) { @@ -710,9 +710,9 @@ protected String toJsonString(ThreatIntelFeedData tifd) throws IOException { return IndexUtilsKt.string(shuffleXContent(tifd.toXContent(builder, ToXContent.EMPTY_PARAMS))); } - private String toJsonString(SATIFSourceConfigDto satifSourceConfigDto) throws IOException { + private String toJsonString(SATIFSourceConfigDto SaTifSourceConfigDto) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); - return IndexUtilsKt.string(shuffleXContent(satifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); + return IndexUtilsKt.string(shuffleXContent(SaTifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); } private String alertingScheduledJobMappings() { diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java index 95cbadf0e..2a59f7781 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -33,7 +33,7 @@ public class SATIFSourceConfigRestApiIT extends SecurityAnalyticsRestTestCase { public void testCreateSATIFSourceConfig() throws IOException { IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); - SATIFSourceConfigDto satifSourceConfigDto = new SATIFSourceConfigDto( + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( null, null, "feedname", @@ -53,7 +53,7 @@ public void testCreateSATIFSourceConfig() throws IOException { List.of("ip", "dns") ); - Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI, Collections.emptyMap(), toHttpEntity(satifSourceConfigDto)); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(SaTifSourceConfigDto)); Assert.assertEquals(201, response.getStatusLine().getStatusCode()); Map responseBody = asMap(response); @@ -62,7 +62,7 @@ public void testCreateSATIFSourceConfig() throws IOException { int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); Assert.assertTrue("incorrect version", createdVersion > 0); - Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.TIF_SOURCE_CONFIG_URI, createdId), response.getHeader("Location")); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); String request = "{\n" + " \"query\" : {\n" + From 2a8c08503f009923d497f26b388a8fdcc99f2d50 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 27 May 2024 10:06:56 -0400 Subject: [PATCH 6/7] fix error message Signed-off-by: Joanne Wang --- .../securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java | 2 +- .../threatIntel/service/SATIFSourceConfigService.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java index 67d846d90..8c91cf0d9 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java @@ -125,7 +125,7 @@ public void onFailure(final Exception e) { stepListener.onResponse(null); return; } - log.error("Failed to create security analytics threat intel job index", e); + log.error("Failed to create security analytics threat intel source config index", e); stepListener.onFailure(e); } })); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 740440fb5..5b8b0fb41 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -3,7 +3,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.StepListener; -import org.opensearch.action.support.WriteRequest; import org.opensearch.common.inject.Inject; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; From 9e231f41ea7577d952b28d2f77abdbb9dfcd8435 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 28 May 2024 17:29:49 -0400 Subject: [PATCH 7/7] moved createIndex invocation and other comments Signed-off-by: Joanne Wang --- .../SecurityAnalyticsPlugin.java | 2 +- .../threatIntel/dao/SATIFSourceConfigDao.java | 77 +++++++++++++------ ...igDao.java => TIFSourceConfigService.java} | 6 +- .../service/SATIFSourceConfigService.java | 44 +++++------ 4 files changed, 76 insertions(+), 53 deletions(-) rename src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/{TIFSourceConfigDao.java => TIFSourceConfigService.java} (77%) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 6cb8138fd..dc97878a2 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -179,7 +179,7 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); - SaTifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool); + SaTifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool, threatIntelLockService); SATIFSourceConfigService SaTifSourceConfigService = new SATIFSourceConfigService(SaTifSourceConfigDao, threatIntelLockService); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java index 8c91cf0d9..dacac650c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java @@ -15,18 +15,18 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; -import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -48,33 +48,67 @@ public class SATIFSourceConfigDao { private final ClusterService clusterService; private final ClusterSettings clusterSettings; private final ThreadPool threadPool; + private final TIFLockService lockService; - public SATIFSourceConfigDao(final Client client, final ClusterService clusterService, ThreadPool threadPool) { + + public SATIFSourceConfigDao(final Client client, final ClusterService clusterService, ThreadPool threadPool, final TIFLockService lockService) { this.client = client; this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); this.threadPool = threadPool; + this.lockService = lockService; } - public void indexTIFSourceConfig(SATIFSourceConfig SaTifSourceConfigDto, + public void indexTIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, TimeValue indexTimeout, + final LockModel lock, final ActionListener actionListener) { - try { - IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(SaTifSourceConfigDto.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .timeout(indexTimeout); - log.debug("Indexing tif source config"); - client.index(indexRequest, ActionListener.wrap(response -> { - log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); - SaTifSourceConfigDto.setId(response.getId()); - actionListener.onResponse(SaTifSourceConfigDto); - }, actionListener::onFailure)); - } catch (Exception e) { - log.error("Exception saving the threat intel source config in index", e); - actionListener.onFailure(e); - } + StepListener createIndexStepListener = new StepListener<>(); + createIndexStepListener.whenComplete(v -> { + try { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(SaTifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + log.debug("Indexing tif source config"); + client.index(indexRequest, ActionListener.wrap(response -> { + log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); + SATIFSourceConfig responseSaTifSourceConfig = createSATIFSourceConfig(SaTifSourceConfig, response); + actionListener.onResponse(responseSaTifSourceConfig); + }, actionListener::onFailure)); + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + }, exception -> { + lockService.releaseLock(lock); + log.error("failed to release lock", exception); + actionListener.onFailure(exception); + }); + createJobIndexIfNotExists(createIndexStepListener); + } + + private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, IndexResponse response) { + return new SATIFSourceConfig( + response.getId(), + SaTifSourceConfig.getVersion(), + SaTifSourceConfig.getName(), + SaTifSourceConfig.getFeedFormat(), + SaTifSourceConfig.getFeedType(), + SaTifSourceConfig.getCreatedByUser(), + SaTifSourceConfig.getCreatedAt(), + SaTifSourceConfig.getEnabledTime(), + SaTifSourceConfig.getLastUpdateTime(), + SaTifSourceConfig.getSchedule(), + SaTifSourceConfig.getState(), + SaTifSourceConfig.getRefreshType(), + SaTifSourceConfig.getLastRefreshedTime(), + SaTifSourceConfig.getLastRefreshedUser(), + SaTifSourceConfig.isEnabled(), + SaTifSourceConfig.getIocMapStore(), + SaTifSourceConfig.getIocTypes() + ); } public ThreadPool getThreadPool() { @@ -131,9 +165,4 @@ public void onFailure(final Exception e) { })); } - // Common utils interface class - IndexTIFSourceConfigResponse indexTIFConfig(SAIndexTIFSourceConfigRequest request, ActionListener listener) { - return null; - } - } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java similarity index 77% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDao.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java index ce925a0f3..9f5438a6e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDao.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java @@ -1,8 +1,10 @@ package org.opensearch.securityanalytics.threatIntel.sacommons; import org.opensearch.core.action.ActionListener; -public interface TIFSourceConfigDao { - IndexTIFSourceConfigResponse indexTIFConfig(IndexTIFSourceConfigRequest request, ActionListener listener); +public abstract class TIFSourceConfigService { + IndexTIFSourceConfigResponse indexTIFConfig(IndexTIFSourceConfigRequest request, ActionListener listener){ + return null; + } // TODO: // update diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 5b8b0fb41..e2bd0400c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -50,32 +50,24 @@ public void createIndexAndSaveTIFSourceConfig( final TimeValue indexTimeout, final ActionListener listener ) { - StepListener createIndexStepListener = new StepListener<>(); - createIndexStepListener.whenComplete(v -> { - try { - SATIFSourceConfig SaTifSourceConfig = convertToSATIFConfig(SaTifSourceConfigDto); - SaTifSourceConfig.setState(TIFJobState.AVAILABLE); - SaTifSourceConfigDao.indexTIFSourceConfig(SaTifSourceConfig, indexTimeout, new ActionListener<>() { - @Override - public void onResponse(SATIFSourceConfig response) { - SaTifSourceConfig.setId(response.getId()); - SaTifSourceConfig.setVersion(response.getVersion()); - listener.onResponse(SaTifSourceConfig); - } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - } catch (Exception e) { - listener.onFailure(e); - } - }, exception -> { - lockService.releaseLock(lock); - log.error("failed to release lock", exception); - listener.onFailure(exception); - }); - SaTifSourceConfigDao.createJobIndexIfNotExists(createIndexStepListener); + try { + SATIFSourceConfig SaTifSourceConfig = convertToSATIFConfig(SaTifSourceConfigDto); + SaTifSourceConfig.setState(TIFJobState.AVAILABLE); + SaTifSourceConfigDao.indexTIFSourceConfig(SaTifSourceConfig, indexTimeout, lock, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig response) { + SaTifSourceConfig.setId(response.getId()); + SaTifSourceConfig.setVersion(response.getVersion()); + listener.onResponse(SaTifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } } /**