From c58d233f0a3f6aa3ecaa52bc0b7b853c1061d27c Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Tue, 3 Oct 2023 17:45:35 -0700 Subject: [PATCH] plug threat intel feed into detector creation Signed-off-by: Surya Sashank Nistala --- .../SecurityAnalyticsPlugin.java | 12 +++--- .../DetectorThreatIntelService.java | 26 +++++++++++- .../ThreatIntelFeedDataService.java | 42 ++++++++++++++----- .../TransportIndexDetectorAction.java | 16 +++++-- .../securityanalytics/TestHelpers.java | 4 +- 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 725593ad9..ccf2f44ab 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -12,12 +12,9 @@ import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.routing.Preference; import org.opensearch.core.action.ActionListener; import org.opensearch.action.ActionRequest; import org.opensearch.core.action.ActionResponse; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; @@ -38,7 +35,6 @@ import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.mapper.Mapper; -import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.ClusterPlugin; import org.opensearch.plugins.EnginePlugin; @@ -49,7 +45,6 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; -import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.action.*; import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService; import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper; @@ -62,6 +57,8 @@ import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.*; +import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.transport.*; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Detector; @@ -129,6 +126,7 @@ public Collection createComponents(Client client, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier) { + builtinLogTypeLoader = new BuiltinLogTypeLoader(); logTypeService = new LogTypeService(client, clusterService, xContentRegistry, builtinLogTypeLoader); detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool); @@ -139,11 +137,13 @@ public Collection createComponents(Client client, mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService); ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool); correlationRuleIndices = new CorrelationRuleIndices(client, clusterService); + ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService.state(), client, indexNameExpressionResolver, xContentRegistry); + DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService); this.client = client; return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, - mapperService, indexTemplateManager, builtinLogTypeLoader + mapperService, indexTemplateManager, builtinLogTypeLoader, threatIntelFeedDataService, detectorThreatIntelService ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java index 604d4e983..0e940988e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java @@ -1,7 +1,10 @@ package org.opensearch.securityanalytics.threatIntel; import org.opensearch.commons.alerting.model.DocLevelQuery; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.util.Collections; import java.util.List; @@ -11,8 +14,14 @@ public class DetectorThreatIntelService { + private final ThreatIntelFeedDataService threatIntelFeedDataService; + + public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) { + this.threatIntelFeedDataService = threatIntelFeedDataService; + } + /** Convert the feed data IOCs into query string query format to create doc level queries. */ - public static DocLevelQuery createDocLevelQueryFromThreatIntelList( + public DocLevelQuery createDocLevelQueryFromThreatIntelList( List tifdList, String docLevelQueryId ) { Set iocs = tifdList.stream().map(ThreatIntelFeedData::getIocValue).collect(Collectors.toSet()); @@ -23,7 +32,7 @@ public static DocLevelQuery createDocLevelQueryFromThreatIntelList( ); } - private static String buildQueryStringQueryWithIocList(Set iocs) { + private String buildQueryStringQueryWithIocList(Set iocs) { StringBuilder sb = new StringBuilder(); for(String ioc : iocs) { @@ -36,4 +45,17 @@ private static String buildQueryStringQueryWithIocList(Set iocs) { } return sb.toString(); } + + public DocLevelQuery createDocLevelQueryFromThreatIntel(Detector detector) { + // for testing validation only. + if(detector.getThreatIntelEnabled() ==false) { + throw new SecurityAnalyticsException( + "trying to create threat intel feed queries when flag to use threat intel is disabled.", + RestStatus.FORBIDDEN, new IllegalArgumentException()); + + } + // TODO: plugin logic to run job for populating threat intel feed data + /*threatIntelFeedDataService.getThreatIntelFeedData("ip_address", );*/ + return null; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java index 9c12fdef7..91d156003 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java @@ -28,25 +28,45 @@ */ public class ThreatIntelFeedDataService { private static final Logger log = LogManager.getLogger(FindingsService.class); + private final ClusterState state; + private final Client client; + private final IndexNameExpressionResolver indexNameExpressionResolver; - public static void getThreatIntelFeedData(ClusterState state, Client client, IndexNameExpressionResolver indexNameExpressionResolver, - String feedName, String iocType, - ActionListener> listener, NamedXContentRegistry xContentRegistry) { - String indexPattern = String.format(".opendsearch-sap-threatintel-%s*", feedName); - String tifdIndex = IndexUtils.getNewIndexByCreationDate(state, indexNameExpressionResolver, indexPattern); + public ThreatIntelFeedDataService( + ClusterState state, + Client client, + IndexNameExpressionResolver indexNameExpressionResolver, + NamedXContentRegistry xContentRegistry) { + this.state = state; + this.client = client; + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.xContentRegistry = xContentRegistry; + } + + private final NamedXContentRegistry xContentRegistry; + + public void getThreatIntelFeedData( + String iocType, + ActionListener> listener + ) { + String tifdIndex = IndexUtils.getNewIndexByCreationDate( + this.state, + this.indexNameExpressionResolver, + ".opendsearch-sap-threatintel*" + ); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("ioc_type", iocType))); SearchRequest searchRequest = new SearchRequest(tifdIndex); searchRequest.source().size(9999); //TODO: convert to scroll searchRequest.source(sourceBuilder); - client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(getTifdList(r, xContentRegistry)), e -> { + client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(getTifdList(r)), e -> { log.error(String.format( - "Failed to fetch threat intel feed data %s from system index %s", feedName, tifdIndex), e); + "Failed to fetch threat intel feed data from system index %s", tifdIndex), e); listener.onFailure(e); })); } - private static List getTifdList(SearchResponse searchResponse, NamedXContentRegistry xContentRegistry) { + private List getTifdList(SearchResponse searchResponse) { List list = new ArrayList<>(); if (searchResponse.getHits().getHits().length != 0) { Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { @@ -57,8 +77,10 @@ private static List getTifdList(SearchResponse searchRespon ); list.add(ThreatIntelFeedData.parse(xcp, hit.getId(), hit.getVersion())); } catch (Exception e) { - log.error(() -> - new ParameterizedMessage("Failed to parse Threat intel feed data doc from hit {}", hit), e); + log.error(() -> new ParameterizedMessage( + "Failed to parse Threat intel feed data doc from hit {}", hit), + e + ); } }); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index abc4c041b..e89c4b519 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -95,6 +95,7 @@ import org.opensearch.securityanalytics.rules.backend.QueryBackend; import org.opensearch.securityanalytics.rules.exceptions.SigmaError; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.DetectorUtils; import org.opensearch.securityanalytics.util.IndexUtils; @@ -154,6 +155,7 @@ public class TransportIndexDetectorAction extends HandledTransportAction DocLevelQuery docLevelQuery = new DocLevelQuery(id, name, actualQuery, tags); docLevelQueries.add(docLevelQuery); } - if(detector.getThreatIntelEnabled()) { - DetectorThreatIntelService + try { + if (detector.getThreatIntelEnabled()) { + DocLevelQuery docLevelQueryFromThreatIntel = detectorThreatIntelService.createDocLevelQueryFromThreatIntel(detector); + docLevelQueries.add(docLevelQueryFromThreatIntel); + } + } catch (Exception e) { + // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data + log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e); } DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries); docLevelMonitorInputs.add(docLevelMonitorInput); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index c6aec7951..c18c54872 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -164,8 +164,8 @@ public static CustomLogType randomCustomLogType(String name, String description, public static ThreatIntelFeedData randomThreatIntelFeedData() { return new ThreatIntelFeedData( "IP_ADDRESS", - "123.442.111.112", - OpenSearchRestTestCase.randomAlphaOfLength(10), + ip, + "alientVault", Instant.now() ); }