fix threat intel integ tests and add update detector logic

Signed-off-by: Surya Sashank Nistala <[email protected]>

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -112,9 +112,6 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
     private BuiltinLogTypeLoader builtinLogTypeLoader;
 
     private LogTypeService logTypeService;
-
-    private Client client;
-
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings){
         return List.of(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"));
@@ -145,14 +142,12 @@ public Collection<Object> createComponents(Client client,
         ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool);
         correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
         ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService, client, indexNameExpressionResolver, xContentRegistry);
-        DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService);
+        DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService, client, xContentRegistry);
         TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService);
         TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader);
         TIFLockService threatIntelLockService = new TIFLockService(clusterService, client);
 
-        this.client = client;
-
-        TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool);
+        TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService);
 
         return List.of(
                 detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,

src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java

@@ -121,7 +121,7 @@ public class SecurityAnalyticsSettings {
 
     // threat intel settings
     public static final Setting<TimeValue> TIF_UPDATE_INTERVAL = Setting.timeSetting(
-            "plugins.security_analytics.threat_intel_timeout",
+            "plugins.security_analytics.threatintel.tifjob.update_interval",
             TimeValue.timeValueHours(24),
             TimeValue.timeValueHours(1),
             Setting.Property.NodeScope,

src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java

@@ -6,10 +6,19 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.action.support.WriteRequest;
 import org.opensearch.client.Client;
-import org.opensearch.common.settings.Settings;
 import org.opensearch.commons.alerting.model.DocLevelQuery;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.securityanalytics.action.IndexDetectorAction;
+import org.opensearch.securityanalytics.action.IndexDetectorRequest;
+import org.opensearch.securityanalytics.action.SearchDetectorAction;
+import org.opensearch.securityanalytics.action.SearchDetectorRequest;
 import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.model.LogType;
 import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
@@ -24,15 +33,22 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.opensearch.securityanalytics.model.Detector.DETECTORS_INDEX;
+import static org.opensearch.securityanalytics.util.DetectorUtils.getDetectors;
+
 
 public class DetectorThreatIntelService {
 
     private static final Logger log = LogManager.getLogger(DetectorThreatIntelService.class);
 
     private final ThreatIntelFeedDataService threatIntelFeedDataService;
+    private final Client client;
+    private final NamedXContentRegistry xContentRegistry;
 
-    public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) {
+    public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService, Client client, NamedXContentRegistry xContentRegistry) {
         this.threatIntelFeedDataService = threatIntelFeedDataService;
+        this.client = client;
+        this.xContentRegistry = xContentRegistry;
     }
 
 
@@ -62,7 +78,7 @@ public List<DocLevelQuery> createDocLevelQueriesFromThreatIntelList(
                 queries.add(new DocLevelQuery(
                         constructId(detector, entry.getKey()), tifdList.get(0).getFeedId(),
                         Collections.emptyList(),
-                        "windows-hostname:(120.85.114.146 OR 103.104.106.223 OR 185.191.246.45 OR 120.86.237.94)",
+                        String.format(query, field),
                         List.of("threat_intel", entry.getKey() /*ioc_type*/)
                 ));
             }
@@ -128,6 +144,29 @@ private static String constructId(Detector detector, String iocType) {
     }
 
     public void updateDetectorsWithLatestThreatIntelRules() {
+        //todo : fix query for fetching detectors with threat intel enabled = true
+//     String searchReq = "{ \"query\": { \"match\": { \"detector.threat_intel_enabled\": true } } }";
+        SearchRequest searchRequest = new SearchRequest(DETECTORS_INDEX);
+        SearchSourceBuilder ssb = searchRequest.source();
+        ssb.size(9999);
+        client.execute(SearchDetectorAction.INSTANCE, new SearchDetectorRequest(new SearchRequest().source(ssb)),
+                ActionListener.wrap(r -> {
+                    List<Detector> detectors = getDetectors(r, xContentRegistry);
+                    detectors.forEach(detector -> {
+                                assert detector.getThreatIntelEnabled();
+                                client.execute(IndexDetectorAction.INSTANCE, new IndexDetectorRequest(
+                                                detector.getId(), WriteRequest.RefreshPolicy.IMMEDIATE,
+                                                RestRequest.Method.PUT,
+                                                detector),
+                                        ActionListener.wrap(
+                                                res -> log.debug("updated {} with latest threat intel info", res.getDetector().getId()),
+                                                e -> log.error(() -> new ParameterizedMessage("Failed to update detector {} with latest threat intel info", detector.getId()), e)));
+                            }
+                    );
+                }, e -> {
+                    log.error("Failed to fetch detectors to update with threat intel queries.", e);
+                }));
+
 
     }
 }

src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java

@@ -1,3 +1,7 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
 package org.opensearch.securityanalytics.threatIntel;
 
 import org.apache.logging.log4j.LogManager;

src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java

@@ -34,7 +34,7 @@
 import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS;
 
 /**
- * Transport action to create tif job
+ * Transport action to create job to fetch threat intel feed data and save IoCs
  */
 public class TransportPutTIFJobAction extends HandledTransportAction<PutTIFJobRequest, AcknowledgedResponse> {
     private static final Logger log = LogManager.getLogger(TransportPutTIFJobAction.class);

src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java

@@ -1,3 +1,7 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
 package org.opensearch.securityanalytics.threatIntel.feedMetadata;
 
 import org.apache.logging.log4j.LogManager;

src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java

@@ -21,6 +21,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.time.Instant;
 
+import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService;
 import org.opensearch.securityanalytics.threatIntel.common.TIFJobState;
 import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
 import org.opensearch.threadpool.ThreadPool;
@@ -55,6 +56,7 @@ public static TIFJobRunner getJobRunnerInstance() {
     private TIFLockService lockService;
     private boolean initialized;
     private ThreadPool threadPool;
+    private DetectorThreatIntelService detectorThreatIntelService;
 
     public void setThreadPool(ThreadPool threadPool) {
         this.threadPool = threadPool;
@@ -69,14 +71,16 @@ public void initialize(
         final TIFJobUpdateService jobSchedulerUpdateService,
         final TIFJobParameterService jobSchedulerParameterService,
         final TIFLockService threatIntelLockService,
-        final ThreadPool threadPool
+        final ThreadPool threadPool,
+        DetectorThreatIntelService detectorThreatIntelService
     ) {
         this.clusterService = clusterService;
         this.jobSchedulerUpdateService = jobSchedulerUpdateService;
         this.jobSchedulerParameterService = jobSchedulerParameterService;
         this.lockService = threatIntelLockService;
         this.threadPool = threadPool;
         this.initialized = true;
+        this.detectorThreatIntelService = detectorThreatIntelService;
     }
 
     @Override
@@ -152,6 +156,9 @@ protected void updateJobParameter(final ScheduledJobParameter jobParameter, fina
             Instant endTime = Instant.now();
             jobSchedulerUpdateService.deleteAllTifdIndices(oldIndices, newFeedIndices);
             jobSchedulerUpdateService.updateJobSchedulerParameterAsSucceeded(newFeedIndices, jobSchedulerParameter, startTime, endTime);
+            if(false == newFeedIndices.isEmpty()) {
+                detectorThreatIntelService.updateDetectorsWithLatestThreatIntelRules();
+            }
         } catch (Exception e) {
             log.error("Failed to update jobSchedulerParameter for {}", jobSchedulerParameter.getName(), e);
             jobSchedulerParameter.getUpdateStats().setLastFailedAt(Instant.now());

src/test/java/org/opensearch/securityanalytics/TestHelpers.java

@@ -1397,7 +1397,7 @@ public static String randomDocWithIpIoc(int severity,  int version, String ioc)
                 "\"AccountType\":\"User\",\n" +
                 "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
                 "\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
-                "\"Opcode\":\"%blahblah\",\n" +
+                "\"Opcode\":\"blahblah\",\n" +
                 "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
                 "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
                 "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
@@ -1409,7 +1409,7 @@ public static String randomDocWithIpIoc(int severity,  int version, String ioc)
                 "\"CommandLine\": \"eachtest\",\n" +
                 "\"Initiated\": \"true\"\n" +
                 "}";
-        return String.format(Locale.ROOT, ioc, doc, severity, version);
+        return String.format(Locale.ROOT, doc, ioc, severity, version);
 
     }
 

src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java

@@ -20,6 +20,7 @@
 import org.opensearch.securityanalytics.model.DetectorRule;
 import org.opensearch.securityanalytics.model.DetectorTrigger;
 import org.opensearch.securityanalytics.model.Rule;
+import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -45,6 +46,7 @@
 import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
 import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE;
 import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList;
+import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME;
 
 public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase {
     /**
@@ -1227,7 +1229,6 @@ public void testCreateDetectorWiththreatIntelDisabled_updateDetectorWithThreatIn
 
         List<Map<String, Object>> monitorRunResults = (List<Map<String, Object>>) entityAsMap(executeResponse).get("monitor_run_results");
         assertEquals(1, monitorRunResults.size());
-
         Map<String, Object> docLevelQueryResults = ((List<Map<String, Object>>) ((Map<String, Object>) monitorRunResults.get(0).get("input_results")).get("results")).get(0);
         int noOfSigmaRuleMatches = docLevelQueryResults.size();
         assertEquals(1, noOfSigmaRuleMatches);

src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java

@@ -89,6 +89,8 @@ public abstract class ThreatIntelTestCase extends RestActionTestCase {
     protected Settings settings;
     private AutoCloseable openMocks;
     @Mock
+    protected DetectorThreatIntelService detectorThreatIntelService;
+    @Mock
     protected TIFJobParameter tifJobParameter;
 
     @Before
@@ -109,6 +111,7 @@ public void prepareThreatIntelTestCase() {
         when(clusterState.routingTable()).thenReturn(routingTable);
         when(ingestService.getClusterService()).thenReturn(clusterService);
         when(threadPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService());
+        detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService, client, xContentRegistry());
     }
 
     @After

src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java

@@ -11,6 +11,7 @@
 import org.opensearch.jobscheduler.spi.JobExecutionContext;
 import org.opensearch.jobscheduler.spi.LockModel;
 import org.opensearch.jobscheduler.spi.ScheduledJobParameter;
+import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService;
 import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase;
 import org.opensearch.securityanalytics.threatIntel.common.TIFJobState;
 import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
@@ -27,7 +28,7 @@ public class TIFJobRunnerTests extends ThreatIntelTestCase {
     @Before
     public void init() {
         TIFJobRunner.getJobRunnerInstance()
-                .initialize(clusterService, tifJobUpdateService, tifJobParameterService, tifLockService, threadPool);
+                .initialize(clusterService, tifJobUpdateService, tifJobParameterService, tifLockService, threadPool, detectorThreatIntelService);
     }
 
     public void testGetJobRunnerInstance_whenCalledAgain_thenReturnSameInstance() {