diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index d64b47528..c04189ad6 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -112,9 +112,6 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private BuiltinLogTypeLoader builtinLogTypeLoader; private LogTypeService logTypeService; - - private Client client; - @Override public Collection getSystemIndexDescriptors(Settings settings){ return List.of(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); @@ -145,14 +142,12 @@ public Collection createComponents(Client client, ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool); correlationRuleIndices = new CorrelationRuleIndices(client, clusterService); ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService, client, indexNameExpressionResolver, xContentRegistry); - DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService); + DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService, client, xContentRegistry); TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); - this.client = client; - - TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool); + TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 48cb49fac..f3e3b2f5d 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -121,7 +121,7 @@ public class SecurityAnalyticsSettings { // threat intel settings public static final Setting TIF_UPDATE_INTERVAL = Setting.timeSetting( - "plugins.security_analytics.threat_intel_timeout", + "plugins.security_analytics.threatintel.tifjob.update_interval", TimeValue.timeValueHours(24), TimeValue.timeValueHours(1), Setting.Property.NodeScope, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java index 3c532d50e..b02adef04 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java @@ -6,10 +6,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; -import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.DocLevelQuery; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.rest.RestRequest; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.action.IndexDetectorAction; +import org.opensearch.securityanalytics.action.IndexDetectorRequest; +import org.opensearch.securityanalytics.action.SearchDetectorAction; +import org.opensearch.securityanalytics.action.SearchDetectorRequest; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; @@ -24,15 +33,22 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.model.Detector.DETECTORS_INDEX; +import static org.opensearch.securityanalytics.util.DetectorUtils.getDetectors; + public class DetectorThreatIntelService { private static final Logger log = LogManager.getLogger(DetectorThreatIntelService.class); private final ThreatIntelFeedDataService threatIntelFeedDataService; + private final Client client; + private final NamedXContentRegistry xContentRegistry; - public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) { + public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService, Client client, NamedXContentRegistry xContentRegistry) { this.threatIntelFeedDataService = threatIntelFeedDataService; + this.client = client; + this.xContentRegistry = xContentRegistry; } @@ -62,7 +78,7 @@ public List createDocLevelQueriesFromThreatIntelList( queries.add(new DocLevelQuery( constructId(detector, entry.getKey()), tifdList.get(0).getFeedId(), Collections.emptyList(), - "windows-hostname:(120.85.114.146 OR 103.104.106.223 OR 185.191.246.45 OR 120.86.237.94)", + String.format(query, field), List.of("threat_intel", entry.getKey() /*ioc_type*/) )); } @@ -128,6 +144,29 @@ private static String constructId(Detector detector, String iocType) { } public void updateDetectorsWithLatestThreatIntelRules() { + //todo : fix query for fetching detectors with threat intel enabled = true +// String searchReq = "{ \"query\": { \"match\": { \"detector.threat_intel_enabled\": true } } }"; + SearchRequest searchRequest = new SearchRequest(DETECTORS_INDEX); + SearchSourceBuilder ssb = searchRequest.source(); + ssb.size(9999); + client.execute(SearchDetectorAction.INSTANCE, new SearchDetectorRequest(new SearchRequest().source(ssb)), + ActionListener.wrap(r -> { + List detectors = getDetectors(r, xContentRegistry); + detectors.forEach(detector -> { + assert detector.getThreatIntelEnabled(); + client.execute(IndexDetectorAction.INSTANCE, new IndexDetectorRequest( + detector.getId(), WriteRequest.RefreshPolicy.IMMEDIATE, + RestRequest.Method.PUT, + detector), + ActionListener.wrap( + res -> log.debug("updated {} with latest threat intel info", res.getDetector().getId()), + e -> log.error(() -> new ParameterizedMessage("Failed to update detector {} with latest threat intel info", detector.getId()), e))); + } + ); + }, e -> { + log.error("Failed to fetch detectors to update with threat intel queries.", e); + })); + } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java index 75a20f1a5..540fc6cde 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.threatIntel; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java index 060e67620..6ad3a04bd 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java @@ -34,7 +34,7 @@ import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; /** - * Transport action to create tif job + * Transport action to create job to fetch threat intel feed data and save IoCs */ public class TransportPutTIFJobAction extends HandledTransportAction { private static final Logger log = LogManager.getLogger(TransportPutTIFJobAction.class); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java index 967d4c936..6b84e9fe9 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.threatIntel.feedMetadata; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/TIFMetadataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/TIFMetadataService.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index ca1f61347..fdb07618d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.time.Instant; +import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.threadpool.ThreadPool; @@ -55,6 +56,7 @@ public static TIFJobRunner getJobRunnerInstance() { private TIFLockService lockService; private boolean initialized; private ThreadPool threadPool; + private DetectorThreatIntelService detectorThreatIntelService; public void setThreadPool(ThreadPool threadPool) { this.threadPool = threadPool; @@ -69,7 +71,8 @@ public void initialize( final TIFJobUpdateService jobSchedulerUpdateService, final TIFJobParameterService jobSchedulerParameterService, final TIFLockService threatIntelLockService, - final ThreadPool threadPool + final ThreadPool threadPool, + DetectorThreatIntelService detectorThreatIntelService ) { this.clusterService = clusterService; this.jobSchedulerUpdateService = jobSchedulerUpdateService; @@ -77,6 +80,7 @@ public void initialize( this.lockService = threatIntelLockService; this.threadPool = threadPool; this.initialized = true; + this.detectorThreatIntelService = detectorThreatIntelService; } @Override @@ -152,6 +156,9 @@ protected void updateJobParameter(final ScheduledJobParameter jobParameter, fina Instant endTime = Instant.now(); jobSchedulerUpdateService.deleteAllTifdIndices(oldIndices, newFeedIndices); jobSchedulerUpdateService.updateJobSchedulerParameterAsSucceeded(newFeedIndices, jobSchedulerParameter, startTime, endTime); + if(false == newFeedIndices.isEmpty()) { + detectorThreatIntelService.updateDetectorsWithLatestThreatIntelRules(); + } } catch (Exception e) { log.error("Failed to update jobSchedulerParameter for {}", jobSchedulerParameter.getName(), e); jobSchedulerParameter.getUpdateStats().setLastFailedAt(Instant.now()); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 9b17c4aa2..bf57e4b06 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1397,7 +1397,7 @@ public static String randomDocWithIpIoc(int severity, int version, String ioc) "\"AccountType\":\"User\",\n" + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + - "\"Opcode\":\"%blahblah\",\n" + + "\"Opcode\":\"blahblah\",\n" + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + @@ -1409,7 +1409,7 @@ public static String randomDocWithIpIoc(int severity, int version, String ioc) "\"CommandLine\": \"eachtest\",\n" + "\"Initiated\": \"true\"\n" + "}"; - return String.format(Locale.ROOT, ioc, doc, severity, version); + return String.format(Locale.ROOT, doc, ioc, severity, version); } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 07e862369..55fea4224 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -20,6 +20,7 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.Rule; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; import java.io.IOException; import java.util.ArrayList; @@ -45,6 +46,7 @@ import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { /** @@ -1227,7 +1229,6 @@ public void testCreateDetectorWiththreatIntelDisabled_updateDetectorWithThreatIn List> monitorRunResults = (List>) entityAsMap(executeResponse).get("monitor_run_results"); assertEquals(1, monitorRunResults.size()); - Map docLevelQueryResults = ((List>) ((Map) monitorRunResults.get(0).get("input_results")).get("results")).get(0); int noOfSigmaRuleMatches = docLevelQueryResults.size(); assertEquals(1, noOfSigmaRuleMatches); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java index a6661b32a..20d36ab2d 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java @@ -89,6 +89,8 @@ public abstract class ThreatIntelTestCase extends RestActionTestCase { protected Settings settings; private AutoCloseable openMocks; @Mock + protected DetectorThreatIntelService detectorThreatIntelService; + @Mock protected TIFJobParameter tifJobParameter; @Before @@ -109,6 +111,7 @@ public void prepareThreatIntelTestCase() { when(clusterState.routingTable()).thenReturn(routingTable); when(ingestService.getClusterService()).thenReturn(clusterService); when(threadPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); + detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService, client, xContentRegistry()); } @After diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java index f54631462..82038a91f 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java @@ -11,6 +11,7 @@ import org.opensearch.jobscheduler.spi.JobExecutionContext; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; @@ -27,7 +28,7 @@ public class TIFJobRunnerTests extends ThreatIntelTestCase { @Before public void init() { TIFJobRunner.getJobRunnerInstance() - .initialize(clusterService, tifJobUpdateService, tifJobParameterService, tifLockService, threadPool); + .initialize(clusterService, tifJobUpdateService, tifJobParameterService, tifLockService, threadPool, detectorThreatIntelService); } public void testGetJobRunnerInstance_whenCalledAgain_thenReturnSameInstance() {