-
Notifications
You must be signed in to change notification settings - Fork 77
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Create TIF Source Config API #1046
Changes from 4 commits
ed41a7e
8be3664
491bbaf
f850464
f41ac16
2a8c085
9e231f4
83b6156
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,6 +30,8 @@ | |
import org.opensearch.commons.alerting.action.AlertingActions; | ||
import org.opensearch.core.common.io.stream.NamedWriteableRegistry; | ||
import org.opensearch.core.xcontent.NamedXContentRegistry; | ||
import org.opensearch.core.xcontent.XContentParser; | ||
import org.opensearch.core.xcontent.XContentParserUtils; | ||
import org.opensearch.env.Environment; | ||
import org.opensearch.env.NodeEnvironment; | ||
import org.opensearch.index.IndexSettings; | ||
|
@@ -63,13 +65,18 @@ | |
import org.opensearch.securityanalytics.model.CustomLogType; | ||
import org.opensearch.securityanalytics.model.ThreatIntelFeedData; | ||
import org.opensearch.securityanalytics.resthandler.*; | ||
import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; | ||
import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; | ||
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; | ||
import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFConfigAction; | ||
import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; | ||
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; | ||
import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; | ||
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; | ||
import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; | ||
import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; | ||
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; | ||
import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; | ||
import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; | ||
import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; | ||
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; | ||
import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; | ||
|
@@ -87,6 +94,7 @@ | |
import org.opensearch.threadpool.ThreadPool; | ||
import org.opensearch.watcher.ResourceWatcherService; | ||
|
||
import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD; | ||
import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; | ||
|
||
public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { | ||
|
@@ -103,9 +111,12 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map | |
public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; | ||
public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; | ||
public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; | ||
|
||
public static final String TIF_BASE_URI = PLUGINS_BASE_URI + "/tif"; | ||
public static final String TIF_SOURCE_CONFIG_URI = PLUGINS_BASE_URI + "/tif/source"; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. threat_intel/source? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. changed to threat_intel/source |
||
public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; | ||
public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; | ||
public static final String JOB_TYPE = "opensearch_sap_job"; | ||
|
||
public static final Map<String, Object> TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true); | ||
|
||
private CorrelationRuleIndices correlationRuleIndices; | ||
|
@@ -129,6 +140,9 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map | |
private BuiltinLogTypeLoader builtinLogTypeLoader; | ||
|
||
private LogTypeService logTypeService; | ||
|
||
private SATIFSourceConfigDao satifSourceConfigDao; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SaTifSourceConfigDao* There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed |
||
|
||
@Override | ||
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings){ | ||
return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); | ||
|
@@ -165,13 +179,16 @@ public Collection<Object> createComponents(Client client, | |
TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); | ||
TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); | ||
TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); | ||
satifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool); | ||
SATIFSourceConfigService satifSourceConfigService = new SATIFSourceConfigService(satifSourceConfigDao, threatIntelLockService); | ||
|
||
|
||
TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); | ||
|
||
return List.of( | ||
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, | ||
mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, | ||
tifJobUpdateService, tifJobParameterService, threatIntelLockService); | ||
tifJobUpdateService, tifJobParameterService, threatIntelLockService, satifSourceConfigDao, satifSourceConfigService); | ||
} | ||
|
||
@Override | ||
|
@@ -211,13 +228,14 @@ public List<RestHandler> getRestHandlers(Settings settings, | |
new RestSearchCorrelationRuleAction(), | ||
new RestIndexCustomLogTypeAction(), | ||
new RestSearchCustomLogTypeAction(), | ||
new RestDeleteCustomLogTypeAction() | ||
new RestDeleteCustomLogTypeAction(), | ||
new RestIndexTIFConfigAction() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. RestIndexTIFSourceConfigAction There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed |
||
); | ||
} | ||
|
||
@Override | ||
public String getJobType() { | ||
return "opensearch_sap_job"; | ||
return JOB_TYPE; | ||
} | ||
|
||
@Override | ||
|
@@ -232,7 +250,21 @@ public ScheduledJobRunner getJobRunner() { | |
|
||
@Override | ||
public ScheduledJobParser getJobParser() { | ||
return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); | ||
return (xcp, id, jobDocVersion) -> { | ||
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); | ||
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { | ||
String fieldName = xcp.currentName(); | ||
xcp.nextToken(); | ||
switch (fieldName) { | ||
case FEED_SOURCE_CONFIG_FIELD: | ||
return SATIFSourceConfig.parse(xcp, id, null); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why is the third param null There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Will address this in a future PR, need to understand what params should be used for parser |
||
default: | ||
log.warn("Unsupported document was indexed"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. log level should be error message should include There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. reworded error message |
||
xcp.skipChildren(); | ||
} | ||
} | ||
return null; | ||
}; | ||
} | ||
|
||
@Override | ||
|
@@ -332,7 +364,8 @@ public List<Setting<?>> getSettings() { | |
new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), | ||
new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), | ||
new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), | ||
new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class) | ||
new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), | ||
new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class) | ||
); | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.securityanalytics.threatIntel.action; | ||
|
||
import org.opensearch.action.ActionType; | ||
|
||
import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; | ||
|
||
/** | ||
* Threat intel tif job creation action | ||
*/ | ||
public class SAIndexTIFSourceConfigAction extends ActionType<SAIndexTIFSourceConfigResponse> { | ||
|
||
public static final SAIndexTIFSourceConfigAction INSTANCE = new SAIndexTIFSourceConfigAction(); | ||
public static final String NAME = INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; | ||
private SAIndexTIFSourceConfigAction() { | ||
super(NAME, SAIndexTIFSourceConfigResponse::new); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.securityanalytics.threatIntel.action; | ||
|
||
import org.opensearch.action.ActionRequest; | ||
import org.opensearch.action.ActionRequestValidationException; | ||
import org.opensearch.action.support.WriteRequest; | ||
import org.opensearch.common.unit.TimeValue; | ||
import org.opensearch.core.common.io.stream.StreamInput; | ||
import org.opensearch.core.common.io.stream.StreamOutput; | ||
import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; | ||
import org.opensearch.rest.RestRequest; | ||
import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; | ||
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; | ||
import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigRequest; | ||
|
||
import java.io.IOException; | ||
import java.sql.Time; | ||
import java.time.Instant; | ||
import java.time.temporal.ChronoUnit; | ||
import java.util.List; | ||
|
||
/** | ||
* Threat intel feed config creation request | ||
*/ | ||
public class SAIndexTIFSourceConfigRequest extends ActionRequest implements IndexTIFSourceConfigRequest { | ||
private static final ParameterValidator VALIDATOR = new ParameterValidator(); | ||
private String tifSourceConfigId; | ||
private final WriteRequest.RefreshPolicy refreshPolicy; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. dont accept param. always IMMEDIATE REFRESH There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. removed param |
||
private final RestRequest.Method method; | ||
private SATIFSourceConfigDto satifSourceConfigDto; | ||
|
||
public SAIndexTIFSourceConfigRequest(String tifSourceConfigId, | ||
WriteRequest.RefreshPolicy refreshPolicy, | ||
RestRequest.Method method, | ||
SATIFSourceConfigDto satifSourceConfigDto) { | ||
super(); | ||
this.tifSourceConfigId = tifSourceConfigId; | ||
this.refreshPolicy = refreshPolicy; | ||
this.method = method; | ||
this.satifSourceConfigDto = satifSourceConfigDto; | ||
} | ||
|
||
public SAIndexTIFSourceConfigRequest(StreamInput sin) throws IOException { | ||
this( | ||
sin.readString(), // tif config id | ||
WriteRequest.RefreshPolicy.readFrom(sin), // refresh policy | ||
sin.readEnum(RestRequest.Method.class), // method | ||
SATIFSourceConfigDto.readFrom(sin) // SA tif config dto | ||
); | ||
} | ||
|
||
@Override | ||
public void writeTo(StreamOutput out) throws IOException { | ||
out.writeString(tifSourceConfigId); | ||
refreshPolicy.writeTo(out); | ||
out.writeEnum(method); | ||
satifSourceConfigDto.writeTo(out); | ||
} | ||
|
||
@Override | ||
public String getTIFConfigId() { | ||
return tifSourceConfigId; | ||
} | ||
|
||
public void setTIFConfigId(String tifConfigId) { | ||
this.tifSourceConfigId = tifConfigId; | ||
} | ||
|
||
@Override | ||
public SATIFSourceConfigDto getTIFConfigDto() { | ||
return satifSourceConfigDto; | ||
} | ||
|
||
public void setTIFConfigDto(SATIFSourceConfigDto saTifConfigDto) { | ||
this.satifSourceConfigDto = saTifConfigDto; | ||
} | ||
|
||
public WriteRequest.RefreshPolicy getRefreshPolicy() { | ||
return refreshPolicy; | ||
} | ||
|
||
@Override | ||
public ActionRequestValidationException validate() { | ||
ActionRequestValidationException errors = new ActionRequestValidationException(); | ||
List<String> errorMsgs = VALIDATOR.validateTIFJobName(satifSourceConfigDto.getName()); | ||
if (errorMsgs.isEmpty() == false) { | ||
errorMsgs.forEach(errors::addValidationError); | ||
} | ||
return errors.validationErrors().isEmpty() ? null : errors; | ||
} | ||
|
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
package org.opensearch.securityanalytics.threatIntel.action; | ||
|
||
import org.opensearch.core.action.ActionResponse; | ||
import org.opensearch.core.common.io.stream.StreamInput; | ||
import org.opensearch.core.common.io.stream.StreamOutput; | ||
import org.opensearch.core.xcontent.ToXContentObject; | ||
import org.opensearch.core.xcontent.XContentBuilder; | ||
import org.opensearch.core.rest.RestStatus; | ||
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; | ||
import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse; | ||
import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; | ||
|
||
import java.io.IOException; | ||
|
||
import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; | ||
import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; | ||
|
||
public class SAIndexTIFSourceConfigResponse extends ActionResponse implements ToXContentObject, IndexTIFSourceConfigResponse { | ||
private final String tifConfigId; | ||
private final Long version; | ||
private final RestStatus status; | ||
private final SATIFSourceConfigDto saTIFConfigDto; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if a word is an a acronym let's make it liek SaTifSourceConfigDto There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed |
||
|
||
public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto tifConfig) { | ||
super(); | ||
this.tifConfigId = id; | ||
this.version = version; | ||
this.status = status; | ||
this.saTIFConfigDto = tifConfig; | ||
} | ||
|
||
public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException { | ||
this( | ||
sin.readString(), // tif config id | ||
sin.readLong(), // version | ||
sin.readEnum(RestStatus.class), // status | ||
SATIFSourceConfigDto.readFrom(sin) // SA tif config dto | ||
); | ||
} | ||
|
||
@Override | ||
public void writeTo(StreamOutput out) throws IOException { | ||
out.writeString(tifConfigId); | ||
out.writeLong(version); | ||
out.writeEnum(status); | ||
saTIFConfigDto.writeTo(out); | ||
} | ||
|
||
@Override | ||
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
builder.startObject() | ||
.field(_ID, tifConfigId) | ||
.field(_VERSION, version); | ||
|
||
builder.startObject("tif_config") | ||
.field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, saTIFConfigDto.getFeedFormat()) | ||
.field(SATIFSourceConfigDto.FEED_NAME_FIELD, saTIFConfigDto.getName()) | ||
.field(SATIFSourceConfigDto.FEED_TYPE_FIELD, saTIFConfigDto.getFeedType()) | ||
.field(SATIFSourceConfigDto.STATE_FIELD, saTIFConfigDto.getState()) | ||
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTIFConfigDto.getEnabledTime()) | ||
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTIFConfigDto.isEnabled()) | ||
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTIFConfigDto.getLastRefreshedTime()) | ||
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTIFConfigDto.getSchedule()) | ||
// source | ||
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTIFConfigDto.getCreatedByUser()) | ||
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTIFConfigDto.getIocTypes()) | ||
.endObject(); | ||
|
||
return builder.endObject(); | ||
} | ||
@Override | ||
public String getTIFConfigId() { | ||
return tifConfigId; | ||
} | ||
@Override | ||
public Long getVersion() { | ||
return version; | ||
} | ||
@Override | ||
public TIFSourceConfigDto getTIFConfigDto() { | ||
return saTIFConfigDto; | ||
} | ||
public RestStatus getStatus() { | ||
return status; | ||
} | ||
|
||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
threat_intel?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
changed to threat_intel