Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create TIF Source Config API #1046

Merged
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
import org.opensearch.commons.alerting.action.AlertingActions;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.index.IndexSettings;
Expand Down Expand Up @@ -63,13 +65,18 @@
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.resthandler.*;
import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFConfigAction;
import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader;
import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService;
Expand All @@ -87,6 +94,7 @@
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD;
import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension {
Expand All @@ -103,9 +111,12 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate";
public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations";
public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules";

public static final String TIF_BASE_URI = PLUGINS_BASE_URI + "/tif";
Copy link
Member

@eirsep eirsep May 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

threat_intel?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

changed to threat_intel

public static final String TIF_SOURCE_CONFIG_URI = PLUGINS_BASE_URI + "/tif/source";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

threat_intel/source?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

changed to threat_intel/source

public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype";
public static final String JOB_INDEX_NAME = ".opensearch-sap--job";
public static final String JOB_TYPE = "opensearch_sap_job";

public static final Map<String, Object> TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true);

private CorrelationRuleIndices correlationRuleIndices;
Expand All @@ -129,6 +140,9 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
private BuiltinLogTypeLoader builtinLogTypeLoader;

private LogTypeService logTypeService;

private SATIFSourceConfigDao satifSourceConfigDao;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SaTifSourceConfigDao*

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed


@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings){
return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"));
Expand Down Expand Up @@ -165,13 +179,16 @@ public Collection<Object> createComponents(Client client,
TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService);
TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader);
TIFLockService threatIntelLockService = new TIFLockService(clusterService, client);
satifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool);
SATIFSourceConfigService satifSourceConfigService = new SATIFSourceConfigService(satifSourceConfigDao, threatIntelLockService);


TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService);

return List.of(
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,
mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService,
tifJobUpdateService, tifJobParameterService, threatIntelLockService);
tifJobUpdateService, tifJobParameterService, threatIntelLockService, satifSourceConfigDao, satifSourceConfigService);
}

@Override
Expand Down Expand Up @@ -211,13 +228,14 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestSearchCorrelationRuleAction(),
new RestIndexCustomLogTypeAction(),
new RestSearchCustomLogTypeAction(),
new RestDeleteCustomLogTypeAction()
new RestDeleteCustomLogTypeAction(),
new RestIndexTIFConfigAction()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

RestIndexTIFSourceConfigAction

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

);
}

@Override
public String getJobType() {
return "opensearch_sap_job";
return JOB_TYPE;
}

@Override
Expand All @@ -232,7 +250,21 @@ public ScheduledJobRunner getJobRunner() {

@Override
public ScheduledJobParser getJobParser() {
return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null);
return (xcp, id, jobDocVersion) -> {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();
switch (fieldName) {
case FEED_SOURCE_CONFIG_FIELD:
return SATIFSourceConfig.parse(xcp, id, null);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is the third param null

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will address this in a future PR, need to understand what params should be used for parser

default:
log.warn("Unsupported document was indexed");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

log level should be error
re-word message to make it more descriptive

message should include job parser failed, $fieldName (value), in security analytics job registration

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reworded error message

xcp.skipChildren();
}
}
return null;
};
}

@Override
Expand Down Expand Up @@ -332,7 +364,8 @@ public List<Setting<?>> getSettings() {
new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class),
new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class),
new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class),
new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class)
new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class),
new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class)
);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionType;

import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME;

/**
* Threat intel tif job creation action
*/
public class SAIndexTIFSourceConfigAction extends ActionType<SAIndexTIFSourceConfigResponse> {

public static final SAIndexTIFSourceConfigAction INSTANCE = new SAIndexTIFSourceConfigAction();
public static final String NAME = INDEX_TIF_SOURCE_CONFIG_ACTION_NAME;
private SAIndexTIFSourceConfigAction() {
super(NAME, SAIndexTIFSourceConfigResponse::new);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.action.support.WriteRequest;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
import org.opensearch.rest.RestRequest;
import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigRequest;

import java.io.IOException;
import java.sql.Time;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;

/**
* Threat intel feed config creation request
*/
public class SAIndexTIFSourceConfigRequest extends ActionRequest implements IndexTIFSourceConfigRequest {
private static final ParameterValidator VALIDATOR = new ParameterValidator();
private String tifSourceConfigId;
private final WriteRequest.RefreshPolicy refreshPolicy;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dont accept param. always IMMEDIATE REFRESH

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed param

private final RestRequest.Method method;
private SATIFSourceConfigDto satifSourceConfigDto;

public SAIndexTIFSourceConfigRequest(String tifSourceConfigId,
WriteRequest.RefreshPolicy refreshPolicy,
RestRequest.Method method,
SATIFSourceConfigDto satifSourceConfigDto) {
super();
this.tifSourceConfigId = tifSourceConfigId;
this.refreshPolicy = refreshPolicy;
this.method = method;
this.satifSourceConfigDto = satifSourceConfigDto;
}

public SAIndexTIFSourceConfigRequest(StreamInput sin) throws IOException {
this(
sin.readString(), // tif config id
WriteRequest.RefreshPolicy.readFrom(sin), // refresh policy
sin.readEnum(RestRequest.Method.class), // method
SATIFSourceConfigDto.readFrom(sin) // SA tif config dto
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(tifSourceConfigId);
refreshPolicy.writeTo(out);
out.writeEnum(method);
satifSourceConfigDto.writeTo(out);
}

@Override
public String getTIFConfigId() {
return tifSourceConfigId;
}

public void setTIFConfigId(String tifConfigId) {
this.tifSourceConfigId = tifConfigId;
}

@Override
public SATIFSourceConfigDto getTIFConfigDto() {
return satifSourceConfigDto;
}

public void setTIFConfigDto(SATIFSourceConfigDto saTifConfigDto) {
this.satifSourceConfigDto = saTifConfigDto;
}

public WriteRequest.RefreshPolicy getRefreshPolicy() {
return refreshPolicy;
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException errors = new ActionRequestValidationException();
List<String> errorMsgs = VALIDATOR.validateTIFJobName(satifSourceConfigDto.getName());
if (errorMsgs.isEmpty() == false) {
errorMsgs.forEach(errors::addValidationError);
}
return errors.validationErrors().isEmpty() ? null : errors;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.core.action.ActionResponse;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse;
import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto;

import java.io.IOException;

import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION;

public class SAIndexTIFSourceConfigResponse extends ActionResponse implements ToXContentObject, IndexTIFSourceConfigResponse {
private final String tifConfigId;
private final Long version;
private final RestStatus status;
private final SATIFSourceConfigDto saTIFConfigDto;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if a word is an a acronym let's make it liek SaTifSourceConfigDto

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed


public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto tifConfig) {
super();
this.tifConfigId = id;
this.version = version;
this.status = status;
this.saTIFConfigDto = tifConfig;
}

public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException {
this(
sin.readString(), // tif config id
sin.readLong(), // version
sin.readEnum(RestStatus.class), // status
SATIFSourceConfigDto.readFrom(sin) // SA tif config dto
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(tifConfigId);
out.writeLong(version);
out.writeEnum(status);
saTIFConfigDto.writeTo(out);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject()
.field(_ID, tifConfigId)
.field(_VERSION, version);

builder.startObject("tif_config")
.field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, saTIFConfigDto.getFeedFormat())
.field(SATIFSourceConfigDto.FEED_NAME_FIELD, saTIFConfigDto.getName())
.field(SATIFSourceConfigDto.FEED_TYPE_FIELD, saTIFConfigDto.getFeedType())
.field(SATIFSourceConfigDto.STATE_FIELD, saTIFConfigDto.getState())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTIFConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.ENABLED_FIELD, saTIFConfigDto.isEnabled())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTIFConfigDto.getLastRefreshedTime())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTIFConfigDto.getSchedule())
// source
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTIFConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTIFConfigDto.getIocTypes())
.endObject();

return builder.endObject();
}
@Override
public String getTIFConfigId() {
return tifConfigId;
}
@Override
public Long getVersion() {
return version;
}
@Override
public TIFSourceConfigDto getTIFConfigDto() {
return saTIFConfigDto;
}
public RestStatus getStatus() {
return status;
}

}
Loading