From 5c733ed23ef4da206f46ffc99f13291694d3ea61 Mon Sep 17 00:00:00 2001 From: Joanne Wang <109310487+jowg-amazon@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:00:06 -0700 Subject: [PATCH] Threat intel feeds job runner and unit tests (#654) * fix doc level query constructor (#651) Signed-off-by: Surya Sashank Nistala * add mapping for indices storing threat intel feed data * fix feed indices mapping * add threat intel feed data dao Signed-off-by: Surya Sashank Nistala * add threatIntelEnabled field in detector. Signed-off-by: Surya Sashank Nistala * add threat intel feed service and searching feeds Signed-off-by: Surya Sashank Nistala * ti feed data to doc level query convertor logic added * plug threat intel feed into detector creation Signed-off-by: Surya Sashank Nistala * Preliminary framework for jobscheduler and datasource (#626) Signed-off-by: Joanne Wang * with listener and processor Signed-off-by: Joanne Wang * removed actions Signed-off-by: Joanne Wang * clean up Signed-off-by: Joanne Wang * added parser Signed-off-by: Joanne Wang * add unit tests Signed-off-by: Joanne Wang * refactored class names Signed-off-by: Joanne Wang * before moving db Signed-off-by: Joanne Wang * after moving db Signed-off-by: Joanne Wang * added actions to plugin and removed user schedule Signed-off-by: Joanne Wang * unit tests Signed-off-by: Joanne Wang * fix build error Signed-off-by: Joanne Wang * changed transport naming Signed-off-by: Joanne Wang --------- Signed-off-by: Surya Sashank Nistala Signed-off-by: Joanne Wang Co-authored-by: Surya Sashank Nistala --- build.gradle | 2 +- .../SecurityAnalyticsPlugin.java | 66 ++- .../SampleExtensionPlugin.java | 161 ++++++ .../SampleExtensionRestHandler.java | 138 +++++ .../sampleextension/SampleJobParameter.java | 153 ++++++ .../sampleextension/SampleJobRunner.java | 149 ++++++ .../settings/SecurityAnalyticsSettings.java | 49 +- .../ThreatIntelFeedDataService.java | 141 ++--- .../threatIntel/ThreatIntelFeedParser.java | 65 +++ .../action/DeleteTIFJobAction.java} | 14 +- .../action/DeleteTIFJobRequest.java} | 16 +- .../threatIntel/action/GetTIFJobAction.java | 26 + .../action/GetTIFJobRequest.java} | 18 +- .../action/GetTIFJobResponse.java} | 40 +- .../action/PutTIFJobAction.java} | 14 +- .../threatIntel/action/PutTIFJobRequest.java | 107 ++++ .../action/TransportDeleteTIFJobAction.java} | 83 ++- .../action/TransportGetTIFJobAction.java | 78 +++ .../action/TransportPutTIFJobAction.java} | 95 ++-- .../action/TransportUpdateTIFJobAction.java | 133 +++++ .../action/UpdateTIFJobAction.java} | 14 +- .../action/UpdateTIFJobRequest.java | 123 +++++ .../threatIntel/common/FeedMetadata.java | 287 ++++++++++ .../common/TIFExecutor.java} | 12 +- .../threatIntel/common/TIFJobState.java | 37 ++ .../common/TIFLockService.java} | 29 +- .../threatIntel/common/TIFMetadata.java | 309 +++++++++++ .../jobscheduler/TIFJobExtension.java} | 15 +- .../jobscheduler/TIFJobParameter.java} | 494 ++++-------------- .../jobscheduler/TIFJobParameterService.java} | 201 ++++--- .../jobscheduler/TIFJobRunner.java | 167 ++++++ .../jobscheduler/TIFJobTask.java} | 4 +- .../jobscheduler/TIFJobUpdateService.java | 287 ++++++++++ .../action/GetDatasourceAction.java | 26 - .../action/GetDatasourceTransportAction.java | 79 --- .../action/PutDatasourceRequest.java | 267 ---------- .../action/RestDeleteDatasourceHandler.java | 48 -- .../action/RestGetDatasourceHandler.java | 44 -- .../action/RestPutDatasourceHandler.java | 71 --- .../action/RestUpdateDatasourceHandler.java | 50 -- .../action/UpdateDatasourceRequest.java | 190 ------- .../UpdateDatasourceTransportAction.java | 179 ------- .../common/DatasourceManifest.java | 168 ------ .../threatintel/common/DatasourceState.java | 37 -- .../common/ParameterValidator.java | 2 +- .../common/ThreatIntelSettings.java | 103 ---- .../jobscheduler/DatasourceRunner.java | 159 ------ .../jobscheduler/DatasourceUpdateService.java | 296 ----------- .../mappings/threat_intel_job_mapping.json | 118 +++++ .../resources/threatIntelFeedInfo/feodo.yml | 6 + .../threatIntel/ThreatIntelTestCase.java | 287 ++++++++++ .../threatIntel/ThreatIntelTestHelper.java | 120 +++++ .../threatIntel/common/TIFMetadataTests.java | 35 ++ .../common/ThreatIntelLockServiceTests.java | 117 +++++ .../jobscheduler/TIFJobExtensionTests.java | 56 ++ .../TIFJobParameterServiceTests.java | 385 ++++++++++++++ .../jobscheduler/TIFJobParameterTests.java | 90 ++++ .../jobscheduler/TIFJobRunnerTests.java | 177 +++++++ .../TIFJobUpdateServiceTests.java | 205 ++++++++ .../sample_invalid_less_than_two_fields.csv | 2 + .../resources/threatIntel/sample_valid.csv | 3 + 61 files changed, 4337 insertions(+), 2510 deletions(-) create mode 100644 src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionPlugin.java create mode 100644 src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionRestHandler.java create mode 100644 src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobParameter.java create mode 100644 src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobRunner.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/PutDatasourceAction.java => threatIntel/action/DeleteTIFJobAction.java} (55%) rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/DeleteDatasourceRequest.java => threatIntel/action/DeleteTIFJobRequest.java} (73%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobAction.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/GetDatasourceRequest.java => threatIntel/action/GetTIFJobRequest.java} (70%) rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/GetDatasourceResponse.java => threatIntel/action/GetTIFJobResponse.java} (59%) rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/DeleteDatasourceAction.java => threatIntel/action/PutTIFJobAction.java} (54%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/DeleteDatasourceTransportAction.java => threatIntel/action/TransportDeleteTIFJobAction.java} (53%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportGetTIFJobAction.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/PutDatasourceTransportAction.java => threatIntel/action/TransportPutTIFJobAction.java} (61%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportUpdateTIFJobAction.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/action/UpdateDatasourceAction.java => threatIntel/action/UpdateTIFJobAction.java} (54%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedMetadata.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/common/ThreatIntelExecutor.java => threatIntel/common/TIFExecutor.java} (71%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/common/ThreatIntelLockService.java => threatIntel/common/TIFLockService.java} (83%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/jobscheduler/DatasourceExtension.java => threatIntel/jobscheduler/TIFJobExtension.java} (60%) rename src/main/java/org/opensearch/securityanalytics/{threatintel/jobscheduler/Datasource.java => threatIntel/jobscheduler/TIFJobParameter.java} (52%) rename src/main/java/org/opensearch/securityanalytics/{threatintel/dao/DatasourceDao.java => threatIntel/jobscheduler/TIFJobParameterService.java} (62%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java rename src/main/java/org/opensearch/securityanalytics/{threatintel/jobscheduler/DatasourceTask.java => threatIntel/jobscheduler/TIFJobTask.java} (78%) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceAction.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceTransportAction.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceRequest.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/RestDeleteDatasourceHandler.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/RestGetDatasourceHandler.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/RestPutDatasourceHandler.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/RestUpdateDatasourceHandler.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceRequest.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceTransportAction.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceManifest.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceState.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelSettings.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceRunner.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceUpdateService.java create mode 100644 src/main/resources/mappings/threat_intel_job_mapping.json create mode 100644 src/main/resources/threatIntelFeedInfo/feodo.yml create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestHelper.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadataTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateServiceTests.java create mode 100644 src/test/resources/threatIntel/sample_invalid_less_than_two_fields.csv create mode 100644 src/test/resources/threatIntel/sample_valid.csv diff --git a/build.gradle b/build.gradle index 49180e6ab..2a958f0b6 100644 --- a/build.gradle +++ b/build.gradle @@ -155,7 +155,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}" implementation "org.antlr:antlr4-runtime:4.10.1" implementation "com.cronutils:cron-utils:9.1.6" - api files("/Users/snistala/Documents/opensearch/common-utils/build/libs/common-utils-3.0.0.0-SNAPSHOT.jar") + api "org.opensearch:common-utils:${common_utils_version}@jar" api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 3e3d6ee07..e9b9382e8 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -4,11 +4,7 @@ */ package org.opensearch.securityanalytics; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,12 +31,8 @@ import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.mapper.Mapper; -import org.opensearch.plugins.ActionPlugin; -import org.opensearch.plugins.ClusterPlugin; -import org.opensearch.plugins.EnginePlugin; -import org.opensearch.plugins.MapperPlugin; -import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.SearchPlugin; +import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.plugins.*; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; @@ -59,6 +51,12 @@ import org.opensearch.securityanalytics.resthandler.*; import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.threatIntel.action.*; +import org.opensearch.securityanalytics.threatIntel.common.TIFExecutor; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; import org.opensearch.securityanalytics.transport.*; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Detector; @@ -70,10 +68,13 @@ import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.RuleIndices; import org.opensearch.securityanalytics.util.RuleTopicIndices; +import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin { +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; + +public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin { private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class); @@ -114,6 +115,18 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private Client client; + @Override + public Collection getSystemIndexDescriptors(Settings settings){ + return List.of(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); + } + + @Override + public List> getExecutorBuilders(Settings settings) { + List> executorBuilders = new ArrayList<>(); + executorBuilders.add(TIFExecutor.executorBuilder(settings)); + return executorBuilders; + } + @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -137,13 +150,21 @@ public Collection createComponents(Client client, mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService); ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool); correlationRuleIndices = new CorrelationRuleIndices(client, clusterService); - ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService, client, indexNameExpressionResolver, xContentRegistry); + ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService.state(), clusterService, client, indexNameExpressionResolver, xContentRegistry); DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService); + TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); + TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService); + TIFExecutor threatIntelExecutor = new TIFExecutor(threadPool); + TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); + this.client = client; + TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelExecutor, threatIntelLockService, threadPool); + return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, - mapperService, indexTemplateManager, builtinLogTypeLoader, threatIntelFeedDataService, detectorThreatIntelService + mapperService, indexTemplateManager, builtinLogTypeLoader, threatIntelFeedDataService, detectorThreatIntelService, + tifJobUpdateService, tifJobParameterService, threatIntelExecutor, threatIntelLockService ); } @@ -245,7 +266,10 @@ public List> getSettings() { SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING, SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA, - SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE + SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, + SecurityAnalyticsSettings.TIFJOB_UPDATE_INTERVAL, + SecurityAnalyticsSettings.BATCH_SIZE, + SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT ); } @@ -276,8 +300,14 @@ public List> getSettings() { new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class), new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), - new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class) - ); + new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), + + new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), + new ActionHandler<>(GetTIFJobAction.INSTANCE, TransportGetTIFJobAction.class), + new ActionHandler<>(UpdateTIFJobAction.INSTANCE, TransportUpdateTIFJobAction.class), + new ActionHandler<>(DeleteTIFJobAction.INSTANCE, TransportDeleteTIFJobAction.class) + + ); } @Override @@ -294,5 +324,5 @@ public void onFailure(Exception e) { log.warn("Failed to initialize LogType config index and builtin log types"); } }); - } + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionPlugin.java b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionPlugin.java new file mode 100644 index 000000000..653653deb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionPlugin.java @@ -0,0 +1,161 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.sampleextension; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.jobscheduler.spi.JobSchedulerExtension; +import org.opensearch.jobscheduler.spi.ScheduledJobParser; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.plugins.ActionPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.RestController; +import org.opensearch.rest.RestHandler; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +/** + * Sample JobScheduler extension plugin. + * + * It use ".scheduler_sample_extension" index to manage its scheduled jobs, and exposes a REST API + * endpoint using {@link SampleExtensionRestHandler}. + * + */ +public class SampleExtensionPlugin extends Plugin implements ActionPlugin, JobSchedulerExtension { + private static final Logger log = LogManager.getLogger(SampleExtensionPlugin.class); + + static final String JOB_INDEX_NAME = ".scheduler_sample_extension"; + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + SampleJobRunner jobRunner = SampleJobRunner.getJobRunnerInstance(); + jobRunner.setClusterService(clusterService); + jobRunner.setThreadPool(threadPool); + jobRunner.setClient(client); + + return Collections.emptyList(); + } + + @Override + public String getJobType() { + return "scheduler_sample_extension"; + } + + @Override + public String getJobIndex() { + return JOB_INDEX_NAME; + } + + @Override + public ScheduledJobRunner getJobRunner() { + return SampleJobRunner.getJobRunnerInstance(); + } + + @Override + public ScheduledJobParser getJobParser() { + return (parser, id, jobDocVersion) -> { + SampleJobParameter jobParameter = new SampleJobParameter(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + + while (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case SampleJobParameter.NAME_FIELD: + jobParameter.setJobName(parser.text()); + break; + case SampleJobParameter.ENABLED_FILED: + jobParameter.setEnabled(parser.booleanValue()); + break; + case SampleJobParameter.ENABLED_TIME_FILED: + jobParameter.setEnabledTime(parseInstantValue(parser)); + break; + case SampleJobParameter.LAST_UPDATE_TIME_FIELD: + jobParameter.setLastUpdateTime(parseInstantValue(parser)); + break; + case SampleJobParameter.SCHEDULE_FIELD: + jobParameter.setSchedule(ScheduleParser.parse(parser)); + break; + case SampleJobParameter.INDEX_NAME_FIELD: + jobParameter.setIndexToWatch(parser.text()); + break; + case SampleJobParameter.LOCK_DURATION_SECONDS: + jobParameter.setLockDurationSeconds(parser.longValue()); + break; + case SampleJobParameter.JITTER: + jobParameter.setJitter(parser.doubleValue()); + break; + default: + XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); + } + } + return jobParameter; + }; + } + + private Instant parseInstantValue(XContentParser parser) throws IOException { + if (XContentParser.Token.VALUE_NULL.equals(parser.currentToken())) { + return null; + } + if (parser.currentToken().isValue()) { + return Instant.ofEpochMilli(parser.longValue()); + } + XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); + return null; + } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return Collections.singletonList(new SampleExtensionRestHandler()); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionRestHandler.java b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionRestHandler.java new file mode 100644 index 000000000..b0ae1299f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionRestHandler.java @@ -0,0 +1,138 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.sampleextension; + +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * A sample rest handler that supports schedule and deschedule job operation + * + * Users need to provide "id", "index", "job_name", and "interval" parameter to schedule + * a job. e.g. + * {@code + * POST /_plugins/scheduler_sample/watch?id=dashboards-job-id&job_name=watch dashboards index&index=.opensearch_dashboards_1&interval=1 + * } + * + * creates a job with id "dashboards-job-id" and job name "watch dashboards index", + * which logs ".opensearch_dashboards_1" index's shards info every 1 minute + * + * Users can remove that job by calling + * {@code DELETE /_plugins/scheduler_sample/watch?id=dashboards-job-id} + */ +public class SampleExtensionRestHandler extends BaseRestHandler { + public static final String WATCH_INDEX_URI = "/_plugins/scheduler_sample/watch"; + + @Override + public String getName() { + return "Sample JobScheduler extension handler"; + } + + @Override + public List routes() { + return Collections.unmodifiableList( + Arrays.asList(new Route(RestRequest.Method.POST, WATCH_INDEX_URI), new Route(RestRequest.Method.DELETE, WATCH_INDEX_URI)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (request.method().equals(RestRequest.Method.POST)) { + // compose SampleJobParameter object from request + String id = request.param("id"); + String indexName = request.param("index"); + String jobName = request.param("job_name"); + String interval = request.param("interval"); + String lockDurationSecondsString = request.param("lock_duration_seconds"); + Long lockDurationSeconds = lockDurationSecondsString != null ? Long.parseLong(lockDurationSecondsString) : null; + String jitterString = request.param("jitter"); + Double jitter = jitterString != null ? Double.parseDouble(jitterString) : null; + + if (id == null || indexName == null) { + throw new IllegalArgumentException("Must specify id and index parameter"); + } + SampleJobParameter jobParameter = new SampleJobParameter( + id, + jobName, + indexName, + new IntervalSchedule(Instant.now(), Integer.parseInt(interval), ChronoUnit.MINUTES), + lockDurationSeconds, + jitter + ); + IndexRequest indexRequest = new IndexRequest().index(SampleExtensionPlugin.JOB_INDEX_NAME) + .id(id) + .source(jobParameter.toXContent(JsonXContent.contentBuilder(), null)) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + return restChannel -> { + // index the job parameter + client.index(indexRequest, new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + try { + RestResponse restResponse = new BytesRestResponse( + RestStatus.OK, + indexResponse.toXContent(JsonXContent.contentBuilder(), null) + ); + restChannel.sendResponse(restResponse); + } catch (IOException e) { + restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); + } + } + + @Override + public void onFailure(Exception e) { + restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); + } + }); + }; + } else if (request.method().equals(RestRequest.Method.DELETE)) { + // delete job parameter doc from index + String id = request.param("id"); + DeleteRequest deleteRequest = new DeleteRequest().index(SampleExtensionPlugin.JOB_INDEX_NAME).id(id); + + return restChannel -> { + client.delete(deleteRequest, new ActionListener() { + @Override + public void onResponse(DeleteResponse deleteResponse) { + restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Job deleted.")); + } + + @Override + public void onFailure(Exception e) { + restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); + } + }); + }; + } else { + return restChannel -> { + restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed.")); + }; + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobParameter.java b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobParameter.java new file mode 100644 index 000000000..1353b47ab --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobParameter.java @@ -0,0 +1,153 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.sampleextension; + +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.Schedule; + +import java.io.IOException; +import java.time.Instant; + +/** + * A sample job parameter. + *

+ * It adds an additional "indexToWatch" field to {@link ScheduledJobParameter}, which stores the index + * the job runner will watch. + */ +public class SampleJobParameter implements ScheduledJobParameter { + public static final String NAME_FIELD = "name"; + public static final String ENABLED_FILED = "enabled"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String ENABLED_TIME_FILED = "enabled_time"; + public static final String ENABLED_TIME_FILED_READABLE = "enabled_time_field"; + public static final String INDEX_NAME_FIELD = "index_name_to_watch"; + public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds"; + public static final String JITTER = "jitter"; + + private String jobName; + private Instant lastUpdateTime; + private Instant enabledTime; + private boolean isEnabled; + private Schedule schedule; + private String indexToWatch; + private Long lockDurationSeconds; + private Double jitter; + + public SampleJobParameter() {} + + public SampleJobParameter(String id, String name, String indexToWatch, Schedule schedule, Long lockDurationSeconds, Double jitter) { + this.jobName = name; + this.indexToWatch = indexToWatch; + this.schedule = schedule; + + Instant now = Instant.now(); + this.isEnabled = true; + this.enabledTime = now; + this.lastUpdateTime = now; + this.lockDurationSeconds = lockDurationSeconds; + this.jitter = jitter; + } + + @Override + public String getName() { + return this.jobName; + } + + @Override + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + + @Override + public Instant getEnabledTime() { + return this.enabledTime; + } + + @Override + public Schedule getSchedule() { + return this.schedule; + } + + @Override + public boolean isEnabled() { + return this.isEnabled; + } + + @Override + public Long getLockDurationSeconds() { + return this.lockDurationSeconds; + } + + @Override + public Double getJitter() { + return jitter; + } + + public String getIndexToWatch() { + return this.indexToWatch; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + + public void setEnabled(boolean enabled) { + isEnabled = enabled; + } + + public void setSchedule(Schedule schedule) { + this.schedule = schedule; + } + + public void setIndexToWatch(String indexToWatch) { + this.indexToWatch = indexToWatch; + } + + public void setLockDurationSeconds(Long lockDurationSeconds) { + this.lockDurationSeconds = lockDurationSeconds; + } + + public void setJitter(Double jitter) { + this.jitter = jitter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD, this.jobName) + .field(ENABLED_FILED, this.isEnabled) + .field(SCHEDULE_FIELD, this.schedule) + .field(INDEX_NAME_FIELD, this.indexToWatch); + if (this.enabledTime != null) { + builder.timeField(ENABLED_TIME_FILED, ENABLED_TIME_FILED_READABLE, this.enabledTime.toEpochMilli()); + } + if (this.lastUpdateTime != null) { + builder.timeField(LAST_UPDATE_TIME_FIELD, LAST_UPDATE_TIME_FIELD_READABLE, this.lastUpdateTime.toEpochMilli()); + } + if (this.lockDurationSeconds != null) { + builder.field(LOCK_DURATION_SECONDS, this.lockDurationSeconds); + } + if (this.jitter != null) { + builder.field(JITTER, this.jitter); + } + builder.endObject(); + return builder; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobRunner.java b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobRunner.java new file mode 100644 index 000000000..0d62738f1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/sampleextension/SampleJobRunner.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.sampleextension; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.jobscheduler.spi.utils.LockService; +import org.opensearch.plugins.Plugin; +import org.opensearch.threadpool.ThreadPool; + +import java.util.List; +import java.util.UUID; + +/** + * A sample job runner class. + * + * The job runner should be a singleton class if it uses OpenSearch client or other objects passed + * from OpenSearch. Because when registering the job runner to JobScheduler plugin, OpenSearch has + * not invoke plugins' createComponents() method. That is saying the plugin is not completely initalized, + * and the OpenSearch {@link Client}, {@link ClusterService} and other objects + * are not available to plugin and this job runner. + * + * So we have to move this job runner intialization to {@link Plugin} createComponents() method, and using + * singleton job runner to ensure we register a usable job runner instance to JobScheduler plugin. + * + * This sample job runner takes the "indexToWatch" from job parameter and logs that index's shards. + */ +public class SampleJobRunner implements ScheduledJobRunner { + + private static final Logger log = LogManager.getLogger(ScheduledJobRunner.class); + + private static SampleJobRunner INSTANCE; + + public static SampleJobRunner getJobRunnerInstance() { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (SampleJobRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new SampleJobRunner(); + return INSTANCE; + } + } + + private ClusterService clusterService; + private ThreadPool threadPool; + private Client client; + + private SampleJobRunner() { + // Singleton class, use getJobRunner method instead of constructor + } + + public void setClusterService(ClusterService clusterService) { + this.clusterService = clusterService; + } + + public void setThreadPool(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void setClient(Client client) { + this.client = client; + } + + @Override + public void runJob(ScheduledJobParameter jobParameter, JobExecutionContext context) { + if (!(jobParameter instanceof SampleJobParameter)) { + throw new IllegalStateException( + "Job parameter is not instance of SampleJobParameter, type: " + jobParameter.getClass().getCanonicalName() + ); + } + + if (this.clusterService == null) { + throw new IllegalStateException("ClusterService is not initialized."); + } + + if (this.threadPool == null) { + throw new IllegalStateException("ThreadPool is not initialized."); + } + + final LockService lockService = context.getLockService(); + + Runnable runnable = () -> { + if (jobParameter.getLockDurationSeconds() != null) { + lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> { + if (lock == null) { + return; + } + + SampleJobParameter parameter = (SampleJobParameter) jobParameter; + StringBuilder msg = new StringBuilder(); + msg.append("Watching index ").append(parameter.getIndexToWatch()).append("\n"); + + List shardRoutingList = this.clusterService.state().routingTable().allShards(parameter.getIndexToWatch()); + for (ShardRouting shardRouting : shardRoutingList) { + msg.append(shardRouting.shardId().getId()) + .append("\t") + .append(shardRouting.currentNodeId()) + .append("\t") + .append(shardRouting.active() ? "active" : "inactive") + .append("\n"); + } + log.info(msg.toString()); + runTaskForIntegrationTests(parameter); + runTaskForLockIntegrationTests(parameter); + + lockService.release( + lock, + ActionListener.wrap(released -> { log.info("Released lock for job {}", jobParameter.getName()); }, exception -> { + throw new IllegalStateException("Failed to release lock."); + }) + ); + }, exception -> { throw new IllegalStateException("Failed to acquire lock."); })); + } + }; + + threadPool.generic().submit(runnable); + } + + private void runTaskForIntegrationTests(SampleJobParameter jobParameter) { + this.client.index( + new IndexRequest(jobParameter.getIndexToWatch()).id(UUID.randomUUID().toString()) + .source("{\"message\": \"message\"}", XContentType.JSON) + ); + } + + private void runTaskForLockIntegrationTests(SampleJobParameter jobParameter) throws InterruptedException { + if (jobParameter.getName().equals("sample-job-lock-test-it")) { + Thread.sleep(180000); + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 4085d7ae2..967bd3165 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -4,10 +4,14 @@ */ package org.opensearch.securityanalytics.settings; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; import java.util.concurrent.TimeUnit; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; -import org.opensearch.securityanalytics.model.FieldMappingDoc; +import org.opensearch.jobscheduler.repackage.com.cronutils.utils.VisibleForTesting; public class SecurityAnalyticsSettings { public static final String CORRELATION_INDEX = "index.correlation"; @@ -117,4 +121,47 @@ public class SecurityAnalyticsSettings { "ecs", Setting.Property.NodeScope, Setting.Property.Dynamic ); + + // threat intel settings + /** + * Default update interval to be used in threat intel tif job creation API + */ + public static final Setting TIFJOB_UPDATE_INTERVAL = Setting.longSetting( + "plugins.security_analytics.threatintel.tifjob.update_interval_in_days", + 1l, + 1l, //todo: change the min value + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Bulk size for indexing threat intel feed data + */ + public static final Setting BATCH_SIZE = Setting.intSetting( + "plugins.security_analytics.threatintel.tifjob.batch_size", + 10000, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Timeout value for threat intel processor + */ + public static final Setting THREAT_INTEL_TIMEOUT = Setting.timeSetting( + "plugins.security_analytics.threat_intel_timeout", + TimeValue.timeValueSeconds(30), + TimeValue.timeValueSeconds(1), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Return all settings of threat intel feature + * @return a list of all settings for threat intel feature + */ + public static final List> settings() { + return List.of(TIFJOB_UPDATE_INTERVAL, BATCH_SIZE, THREAT_INTEL_TIMEOUT); + } + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java index 1a7001725..b01d602b3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java @@ -1,13 +1,10 @@ package org.opensearch.securityanalytics.threatIntel; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; -import org.opensearch.SpecialPermission; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.bulk.BulkRequest; @@ -22,7 +19,6 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.SuppressForbidden; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -38,43 +34,31 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.findings.FindingsService; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceManifest; +import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelSettings; -import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import org.opensearch.securityanalytics.threatIntel.common.Constants; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.URL; -import java.net.URLConnection; import java.nio.charset.StandardCharsets; -import java.security.AccessController; -import java.security.PrivilegedAction; +import java.time.Instant; import java.util.*; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; /** * Service to handle CRUD operations on Threat Intel Feed Data */ public class ThreatIntelFeedDataService { private static final Logger log = LogManager.getLogger(FindingsService.class); - private static final String SCHEMA_VERSION = "schema_version"; - private static final String IOC_TYPE = "ioc_type"; - private static final String IOC_VALUE = "ioc_value"; - private static final String FEED_ID = "feed_id"; - private static final String TIMESTAMP = "timestamp"; - private static final String TYPE = "type"; - private static final String DATA_FIELD_NAME = "_data"; + private final ClusterState state; private final Client client; private final IndexNameExpressionResolver indexNameExpressionResolver; @@ -95,16 +79,20 @@ public class ThreatIntelFeedDataService { true ); private final ClusterService clusterService; + private final ClusterSettings clusterSettings; public ThreatIntelFeedDataService( + ClusterState state, ClusterService clusterService, Client client, IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry) { + this.state = state; this.client = client; this.indexNameExpressionResolver = indexNameExpressionResolver; this.xContentRegistry = xContentRegistry; this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); } private final NamedXContentRegistry xContentRegistry; @@ -150,6 +138,9 @@ private List getTifdList(SearchResponse searchResponse) { return list; } + + + /** * Create an index for a threat intel feed * @@ -167,28 +158,13 @@ public void createIndexIfNotExists(final String indexName) { .mapping(getIndexMapping()); StashedThreadContext.run( client, - () -> client.admin().indices().create(createIndexRequest).actionGet(this.clusterService.getClusterSettings().get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT)) + () -> client.admin().indices().create(createIndexRequest).actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)) ); } - private void freezeIndex(final String indexName) { - ClusterSettings clusterSettings = this.clusterService.getClusterSettings(); - TimeValue timeout = this.clusterService.getClusterSettings().get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT); - StashedThreadContext.run(client, () -> { - client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); - client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); - client.admin() - .indices() - .prepareUpdateSettings(indexName) - .setSettings(INDEX_SETTING_TO_FREEZE) - .execute() - .actionGet(clusterSettings.get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT)); - }); - } - private String getIndexMapping() { try { - try (InputStream is = DatasourceDao.class.getResourceAsStream("/mappings/threat_intel_feed_mapping.json")) { // TODO: check Datasource dao and this mapping + try (InputStream is = TIFJobParameterService.class.getResourceAsStream("/mappings/threat_intel_feed_mapping.json")) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { return reader.lines().map(String::trim).collect(Collectors.joining()); } @@ -199,74 +175,48 @@ private String getIndexMapping() { } } - /** - * Create CSVParser of a threat intel feed - * - * @param manifest Datasource manifest - * @return CSVParser for threat intel feed - */ - @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") - public CSVParser getDatabaseReader(final DatasourceManifest manifest) { - SpecialPermission.check(); - return AccessController.doPrivileged((PrivilegedAction) () -> { - try { - URL url = new URL(manifest.getUrl()); - return internalGetDatabaseReader(manifest, url.openConnection()); - } catch (IOException e) { - log.error("Exception: failed to read threat intel feed data from {}",manifest.getUrl(), e); - throw new OpenSearchException("failed to read threat intel feed data from {}", manifest.getUrl(), e); - } - }); - } - - @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") // TODO: update this function because no zip file... - protected CSVParser internalGetDatabaseReader(final DatasourceManifest manifest, final URLConnection connection) throws IOException { - connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); - ZipInputStream zipIn = new ZipInputStream(connection.getInputStream()); - ZipEntry zipEntry = zipIn.getNextEntry(); - while (zipEntry != null) { - if (zipEntry.getName().equalsIgnoreCase(manifest.getDbName()) == false) { - zipEntry = zipIn.getNextEntry(); - continue; - } - return new CSVParser(new BufferedReader(new InputStreamReader(zipIn)), CSVFormat.RFC4180); - } - throw new IllegalArgumentException( - String.format(Locale.ROOT, "database file [%s] does not exist in the zip file [%s]", manifest.getDbName(), manifest.getUrl()) - ); - } - /** * Puts threat intel feed from CSVRecord iterator into a given index in bulk * - * @param indexName Index name to puts the TIF data + * @param indexName Index name to save the threat intel feed * @param fields Field name matching with data in CSVRecord in order * @param iterator TIF data to insert * @param renewLock Runnable to renew lock */ - public void saveThreatIntelFeedData( + public void saveThreatIntelFeedDataCSV( final String indexName, final String[] fields, final Iterator iterator, - final Runnable renewLock -// final ThreatIntelFeedData threatIntelFeedData + final Runnable renewLock, + final TIFMetadata tifMetadata ) throws IOException { if (indexName == null || fields == null || iterator == null || renewLock == null){ - throw new IllegalArgumentException("Fields cannot be null"); + throw new IllegalArgumentException("Parameters cannot be null, failed to save threat intel feed data"); } - ClusterSettings clusterSettings = this.clusterService.getClusterSettings(); - TimeValue timeout = clusterSettings.get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT); - Integer batchSize = clusterSettings.get(ThreatIntelSettings.BATCH_SIZE); + + TimeValue timeout = clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT); + Integer batchSize = clusterSettings.get(SecurityAnalyticsSettings.BATCH_SIZE); final BulkRequest bulkRequest = new BulkRequest(); Queue requests = new LinkedList<>(); for (int i = 0; i < batchSize; i++) { requests.add(Requests.indexRequest(indexName)); } + while (iterator.hasNext()) { CSVRecord record = iterator.next(); -// XContentBuilder tifData = threatIntelFeedData.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); + String iocType = tifMetadata.getFeedType(); + if (tifMetadata.getContainedIocs().get(0) == "ip") { //TODO: dynamically get the type + iocType = "ip"; + } + Integer colNum = Integer.parseInt(tifMetadata.getIocCol()); + String iocValue = record.values()[colNum]; + String feedId = tifMetadata.getFeedId(); + Instant timestamp = Instant.now(); + + ThreatIntelFeedData threatIntelFeedData = new ThreatIntelFeedData(iocType, iocValue, feedId, timestamp); + XContentBuilder tifData = threatIntelFeedData.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); IndexRequest indexRequest = (IndexRequest) requests.poll(); -// indexRequest.source(tifData); + indexRequest.source(tifData); indexRequest.id(record.get(0)); bulkRequest.add(indexRequest); if (iterator.hasNext() == false || bulkRequest.requests().size() == batchSize) { @@ -286,12 +236,25 @@ public void saveThreatIntelFeedData( freezeIndex(indexName); } + private void freezeIndex(final String indexName) { + TimeValue timeout = clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT); + StashedThreadContext.run(client, () -> { + client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); + client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); + client.admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(INDEX_SETTING_TO_FREEZE) + .execute() + .actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)); + }); + } + public void deleteThreatIntelDataIndex(final String index) { deleteThreatIntelDataIndex(Arrays.asList(index)); } public void deleteThreatIntelDataIndex(final List indices) { - ClusterSettings clusterSettings = this.clusterService.getClusterSettings(); if (indices == null || indices.isEmpty()) { return; } @@ -314,11 +277,11 @@ public void deleteThreatIntelDataIndex(final List indices) { .prepareDelete(indices.toArray(new String[0])) .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) .execute() - .actionGet(clusterSettings.get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT)) + .actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)) ); if (response.isAcknowledged() == false) { - throw new OpenSearchException("failed to delete data[{}] in datasource", String.join(",", indices)); + throw new OpenSearchException("failed to delete data[{}]", String.join(",", indices)); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java new file mode 100644 index 000000000..ab4477a44 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java @@ -0,0 +1,65 @@ +package org.opensearch.securityanalytics.threatIntel; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.SpecialPermission; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.common.Constants; +import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; + +import java.io.*; +import java.net.URL; +import java.net.URLConnection; +import java.security.AccessController; +import java.security.PrivilegedAction; + +//Parser helper class +public class ThreatIntelFeedParser { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + + /** + * Create CSVParser of a threat intel feed + * + * @param tifMetadata Threat intel feed metadata + * @return parser for threat intel feed + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") + public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadata) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + URL url = new URL(tifMetadata.getUrl()); + URLConnection connection = url.openConnection(); + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); + } catch (IOException e) { + log.error("Exception: failed to read threat intel feed data from {}",tifMetadata.getUrl(), e); + throw new OpenSearchException("failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); + } + }); + } + + /** + * Validate header + * + * 1. header should not be null + * 2. the number of values in header should be more than one + * + * @param header the header + * @return CSVRecord the input header + */ + public static CSVRecord validateHeader(CSVRecord header) { + if (header == null) { + throw new OpenSearchException("threat intel feed database is empty"); + } + if (header.values().length < 2) { + throw new OpenSearchException("threat intel feed database should have at least two fields"); + } + return header; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobAction.java similarity index 55% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobAction.java index 6a6acb9ed..d0fd0bee4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobAction.java @@ -9,19 +9,19 @@ import org.opensearch.action.support.master.AcknowledgedResponse; /** - * Threat intel datasource creation action + * Threat intel tif job delete action */ -public class PutDatasourceAction extends ActionType { +public class DeleteTIFJobAction extends ActionType { /** - * Put datasource action instance + * Delete tif job action instance */ - public static final PutDatasourceAction INSTANCE = new PutDatasourceAction(); + public static final DeleteTIFJobAction INSTANCE = new DeleteTIFJobAction(); /** - * Put datasource action name + * Delete tif job action name */ - public static final String NAME = "cluster:admin/security_analytics/datasource/put"; + public static final String NAME = "cluster:admin/security_analytics/tifjob/delete"; - private PutDatasourceAction() { + private DeleteTIFJobAction() { super(NAME, AcknowledgedResponse::new); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequest.java similarity index 73% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceRequest.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequest.java index 654b93985..54e41126f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequest.java @@ -14,14 +14,14 @@ import java.io.IOException; /** - * Threat intel datasource delete request + * Threat intel feed job delete request */ -public class DeleteDatasourceRequest extends ActionRequest { +public class DeleteTIFJobRequest extends ActionRequest { private static final ParameterValidator VALIDATOR = new ParameterValidator(); /** - * @param name the datasource name - * @return the datasource name + * @param name the TIF job name + * @return the TIF job name */ private String name; @@ -31,21 +31,21 @@ public class DeleteDatasourceRequest extends ActionRequest { * @param in the stream input * @throws IOException IOException */ - public DeleteDatasourceRequest(final StreamInput in) throws IOException { + public DeleteTIFJobRequest(final StreamInput in) throws IOException { super(in); this.name = in.readString(); } - public DeleteDatasourceRequest(final String name) { + public DeleteTIFJobRequest(final String name) { this.name = name; } @Override public ActionRequestValidationException validate() { ActionRequestValidationException errors = null; - if (VALIDATOR.validateDatasourceName(name).isEmpty() == false) { + if (VALIDATOR.validateTIFJobName(name).isEmpty() == false) { errors = new ActionRequestValidationException(); - errors.addValidationError("no such datasource exist"); + errors.addValidationError("no such job exist"); } return errors; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobAction.java new file mode 100644 index 000000000..8f1034d94 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobAction.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +/** + * Threat intel tif job get action + */ +public class GetTIFJobAction extends ActionType { + /** + * Get tif job action instance + */ + public static final GetTIFJobAction INSTANCE = new GetTIFJobAction(); + /** + * Get tif job action name + */ + public static final String NAME = "cluster:admin/security_analytics/tifjob/get"; + + private GetTIFJobAction() { + super(NAME, GetTIFJobResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobRequest.java similarity index 70% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceRequest.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobRequest.java index 16f36b08e..c40e1f747 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobRequest.java @@ -13,24 +13,24 @@ import java.io.IOException; /** - * threat intel datasource get request + * threat intel tif job get request */ -public class GetDatasourceRequest extends ActionRequest { +public class GetTIFJobRequest extends ActionRequest { /** - * @param names the datasource names - * @return the datasource names + * @param names the tif job names + * @return the tif job names */ private String[] names; /** - * Constructs a new get datasource request with a list of datasources. + * Constructs a new get tif job request with a list of tif jobs. * - * If the list of datasources is empty or it contains a single element "_all", all registered datasources + * If the list of tif jobs is empty or it contains a single element "_all", all registered tif jobs * are returned. * - * @param names list of datasource names + * @param names list of tif job names */ - public GetDatasourceRequest(final String[] names) { + public GetTIFJobRequest(final String[] names) { this.names = names; } @@ -39,7 +39,7 @@ public GetDatasourceRequest(final String[] names) { * @param in the stream input * @throws IOException IOException */ - public GetDatasourceRequest(final StreamInput in) throws IOException { + public GetTIFJobRequest(final StreamInput in) throws IOException { super(in); this.names = in.readStringArray(); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobResponse.java similarity index 59% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceResponse.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobResponse.java index d404ad728..507f1f4ee 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobResponse.java @@ -11,34 +11,32 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; import java.io.IOException; import java.time.Instant; import java.util.List; /** - * threat intel datasource get request + * threat intel tif job get request */ -public class GetDatasourceResponse extends ActionResponse implements ToXContentObject { - private static final ParseField FIELD_NAME_DATASOURCES = new ParseField("datasources"); +public class GetTIFJobResponse extends ActionResponse implements ToXContentObject { + private static final ParseField FIELD_NAME_TIFJOBS = new ParseField("tifjobs"); private static final ParseField FIELD_NAME_NAME = new ParseField("name"); private static final ParseField FIELD_NAME_STATE = new ParseField("state"); - private static final ParseField FIELD_NAME_ENDPOINT = new ParseField("endpoint"); private static final ParseField FIELD_NAME_UPDATE_INTERVAL = new ParseField("update_interval_in_days"); private static final ParseField FIELD_NAME_NEXT_UPDATE_AT = new ParseField("next_update_at_in_epoch_millis"); private static final ParseField FIELD_NAME_NEXT_UPDATE_AT_READABLE = new ParseField("next_update_at"); - private static final ParseField FIELD_NAME_DATABASE = new ParseField("database"); private static final ParseField FIELD_NAME_UPDATE_STATS = new ParseField("update_stats"); - private List datasources; + private List tifJobParameters; /** * Default constructor * - * @param datasources List of datasources + * @param tifJobParameters List of tifJobParameters */ - public GetDatasourceResponse(final List datasources) { - this.datasources = datasources; + public GetTIFJobResponse(final List tifJobParameters) { + this.tifJobParameters = tifJobParameters; } /** @@ -46,32 +44,30 @@ public GetDatasourceResponse(final List datasources) { * * @param in the stream input */ - public GetDatasourceResponse(final StreamInput in) throws IOException { - datasources = in.readList(Datasource::new); + public GetTIFJobResponse(final StreamInput in) throws IOException { + tifJobParameters = in.readList(TIFJobParameter::new); } @Override public void writeTo(final StreamOutput out) throws IOException { - out.writeList(datasources); + out.writeList(tifJobParameters); } @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); - builder.startArray(FIELD_NAME_DATASOURCES.getPreferredName()); - for (Datasource datasource : datasources) { + builder.startArray(FIELD_NAME_TIFJOBS.getPreferredName()); + for (TIFJobParameter tifJobParameter : tifJobParameters) { builder.startObject(); - builder.field(FIELD_NAME_NAME.getPreferredName(), datasource.getName()); - builder.field(FIELD_NAME_STATE.getPreferredName(), datasource.getState()); - builder.field(FIELD_NAME_ENDPOINT.getPreferredName(), datasource.getEndpoint()); - builder.field(FIELD_NAME_UPDATE_INTERVAL.getPreferredName(), datasource.getSchedule()); //TODO + builder.field(FIELD_NAME_NAME.getPreferredName(), tifJobParameter.getName()); + builder.field(FIELD_NAME_STATE.getPreferredName(), tifJobParameter.getState()); + builder.field(FIELD_NAME_UPDATE_INTERVAL.getPreferredName(), tifJobParameter.getSchedule()); //TODO builder.timeField( FIELD_NAME_NEXT_UPDATE_AT.getPreferredName(), FIELD_NAME_NEXT_UPDATE_AT_READABLE.getPreferredName(), - datasource.getSchedule().getNextExecutionTime(Instant.now()).toEpochMilli() + tifJobParameter.getSchedule().getNextExecutionTime(Instant.now()).toEpochMilli() ); - builder.field(FIELD_NAME_DATABASE.getPreferredName(), datasource.getDatabase()); - builder.field(FIELD_NAME_UPDATE_STATS.getPreferredName(), datasource.getUpdateStats()); + builder.field(FIELD_NAME_UPDATE_STATS.getPreferredName(), tifJobParameter.getUpdateStats()); builder.endObject(); } builder.endArray(); diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobAction.java similarity index 54% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobAction.java index 35effc4b7..01863f862 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobAction.java @@ -9,19 +9,19 @@ import org.opensearch.action.support.master.AcknowledgedResponse; /** - * Threat intel datasource delete action + * Threat intel tif job creation action */ -public class DeleteDatasourceAction extends ActionType { +public class PutTIFJobAction extends ActionType { /** - * Delete datasource action instance + * Put tif job action instance */ - public static final DeleteDatasourceAction INSTANCE = new DeleteDatasourceAction(); + public static final PutTIFJobAction INSTANCE = new PutTIFJobAction(); /** - * Delete datasource action name + * Put tif job action name */ - public static final String NAME = "cluster:admin/security_analytics/datasource/delete"; + public static final String NAME = "cluster:admin/security_analytics/tifjob/put"; - private DeleteDatasourceAction() { + private PutTIFJobAction() { super(NAME, AcknowledgedResponse::new); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java new file mode 100644 index 000000000..1662979d2 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java @@ -0,0 +1,107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; + +import java.io.IOException; +import java.util.List; + +/** + * Threat intel tif job creation request + */ +public class PutTIFJobRequest extends ActionRequest { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + + public static final ParseField NAME_FIELD = new ParseField("name_FIELD"); +// public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); + private static final ParameterValidator VALIDATOR = new ParameterValidator(); + + /** + * @param name the tif job name + * @return the tif job name + */ + private String name; + + /** + * @param updateInterval update interval of a tif job + * @return update interval of a tif job + */ + private TimeValue updateInterval; + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public TimeValue getUpdateInterval() { + return this.updateInterval; + } + + public void setUpdateInterval(TimeValue timeValue) { + this.updateInterval = timeValue; + } + + /** + * Parser of a tif job + */ + public static final ObjectParser PARSER; + static { + PARSER = new ObjectParser<>("put_tifjob"); + PARSER.declareString((request, val) -> request.setName(val), NAME_FIELD); +// PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); + } + + /** + * Default constructor + * @param name name of a tif job + */ + public PutTIFJobRequest(final String name) { + this.name = name; + } + + /** + * Constructor with stream input + * @param in the stream input + * @throws IOException IOException + */ + public PutTIFJobRequest(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.updateInterval = in.readTimeValue(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeTimeValue(updateInterval); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + List errorMsgs = VALIDATOR.validateTIFJobName(name); + if (errorMsgs.isEmpty() == false) { + errorMsgs.stream().forEach(msg -> errors.addValidationError(msg)); + } + return errors.validationErrors().isEmpty() ? null : errors; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java similarity index 53% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceTransportAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java index 5ff65a945..638893f2e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java @@ -15,14 +15,13 @@ import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; - import org.opensearch.ingest.IngestService; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceState; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelLockService; -import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao; import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -30,17 +29,16 @@ import java.io.IOException; /** - * Transport action to delete datasource + * Transport action to delete tif job */ -public class DeleteDatasourceTransportAction extends HandledTransportAction { +public class TransportDeleteTIFJobAction extends HandledTransportAction { private static final Logger log = LogManager.getLogger(DetectorTrigger.class); private static final long LOCK_DURATION_IN_SECONDS = 300l; - private final ThreatIntelLockService lockService; + private final TIFLockService lockService; private final IngestService ingestService; - private final DatasourceDao datasourceDao; + private final TIFJobParameterService tifJobParameterService; private final ThreatIntelFeedDataService threatIntelFeedDataService; -// private final Ip2GeoProcessorDao ip2GeoProcessorDao; private final ThreadPool threadPool; /** @@ -49,37 +47,35 @@ public class DeleteDatasourceTransportAction extends HandledTransportAction listener) { + protected void doExecute(final Task task, final DeleteTIFJobRequest request, final ActionListener listener) { lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { listener.onFailure( @@ -93,13 +89,13 @@ protected void doExecute(final Task task, final DeleteDatasourceRequest request, // TODO: makes every sub-methods as async call to avoid using a thread in generic pool threadPool.generic().submit(() -> { try { - deleteDatasource(request.getName()); + deleteTIFJob(request.getName()); lockService.releaseLock(lock); listener.onResponse(new AcknowledgedResponse(true)); } catch (Exception e) { lockService.releaseLock(lock); listener.onFailure(e); - log.error("delete data source failed",e); + log.error("delete tif job failed",e); } }); } catch (Exception e) { @@ -110,43 +106,24 @@ protected void doExecute(final Task task, final DeleteDatasourceRequest request, }, exception -> { listener.onFailure(exception); })); } - protected void deleteDatasource(final String datasourceName) throws IOException { - Datasource datasource = datasourceDao.getDatasource(datasourceName); - if (datasource == null) { - throw new ResourceNotFoundException("no such datasource exist"); + protected void deleteTIFJob(final String tifJobName) throws IOException { + TIFJobParameter tifJobParameter = tifJobParameterService.getJobParameter(tifJobName); + if (tifJobParameter == null) { + throw new ResourceNotFoundException("no such tifJobParameter exist"); } - DatasourceState previousState = datasource.getState(); -// setDatasourceStateAsDeleting(datasource); + TIFJobState previousState = tifJobParameter.getState(); + tifJobParameter.setState(TIFJobState.DELETING); + tifJobParameterService.updateJobSchedulerParameter(tifJobParameter); try { - threatIntelFeedDataService.deleteThreatIntelDataIndex(datasource.getIndices()); + threatIntelFeedDataService.deleteThreatIntelDataIndex(tifJobParameter.getIndices()); } catch (Exception e) { - if (previousState.equals(datasource.getState()) == false) { - datasource.setState(previousState); - datasourceDao.updateDatasource(datasource); + if (previousState.equals(tifJobParameter.getState()) == false) { + tifJobParameter.setState(previousState); + tifJobParameterService.updateJobSchedulerParameter(tifJobParameter); } throw e; } - datasourceDao.deleteDatasource(datasource); + tifJobParameterService.deleteTIFJobParameter(tifJobParameter); } - -// private void setDatasourceStateAsDeleting(final Datasource datasource) { -// if (datasourceDao.getProcessors(datasource.getName()).isEmpty() == false) { -// throw new OpenSearchStatusException("datasource is being used by one of processors", RestStatus.BAD_REQUEST); -// } -// -// DatasourceState previousState = datasource.getState(); -// datasource.setState(DatasourceState.DELETING); -// datasourceDao.updateDatasource(datasource); -// -// // Check again as processor might just have been created. -// // If it fails to update the state back to the previous state, the new processor -// // will fail to convert an ip to a geo data. -// // In such case, user have to delete the processor and delete this datasource again. -// if (datasourceDao.getProcessors(datasource.getName()).isEmpty() == false) { -// datasource.setState(previousState); -// datasourceDao.updateDatasource(datasource); -// throw new OpenSearchStatusException("datasource is being used by one of processors", RestStatus.BAD_REQUEST); -// } -// } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportGetTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportGetTIFJobAction.java new file mode 100644 index 000000000..1f884eea1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportGetTIFJobAction.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.OpenSearchException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import java.util.Collections; +import java.util.List; + +/** + * Transport action to get tif job + */ +public class TransportGetTIFJobAction extends HandledTransportAction { + private final TIFJobParameterService tifJobParameterService; + + /** + * Default constructor + * @param transportService the transport service + * @param actionFilters the action filters + * @param tifJobParameterService the tif job parameter service facade + */ + @Inject + public TransportGetTIFJobAction( + final TransportService transportService, + final ActionFilters actionFilters, + final TIFJobParameterService tifJobParameterService + ) { + super(GetTIFJobAction.NAME, transportService, actionFilters, GetTIFJobRequest::new); + this.tifJobParameterService = tifJobParameterService; + } + + @Override + protected void doExecute(final Task task, final GetTIFJobRequest request, final ActionListener listener) { + if (shouldGetAllTIFJobs(request)) { + // We don't expect too many tif jobs. Therefore, querying all tif jobs without pagination should be fine. + tifJobParameterService.getAllTIFJobParameters(newActionListener(listener)); + } else { + tifJobParameterService.getTIFJobParameters(request.getNames(), newActionListener(listener)); + } + } + + private boolean shouldGetAllTIFJobs(final GetTIFJobRequest request) { + if (request.getNames() == null) { + throw new OpenSearchException("names in a request should not be null"); + } + return request.getNames().length == 0 || (request.getNames().length == 1 && "_all".equals(request.getNames()[0])); + } + + protected ActionListener> newActionListener(final ActionListener listener) { + return new ActionListener<>() { + @Override + public void onResponse(final List tifJobParameters) { + listener.onResponse(new GetTIFJobResponse(tifJobParameters)); + } + + @Override + public void onFailure(final Exception e) { + if (e instanceof IndexNotFoundException) { + listener.onResponse(new GetTIFJobResponse(Collections.emptyList())); + return; + } + listener.onFailure(e); + } + }; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java similarity index 61% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceTransportAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java index f1f87c4c5..c32a64c1c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java @@ -5,12 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.action; -import static org.opensearch.securityanalytics.threatIntel.common.ThreatIntelLockService.LOCK_DURATION_IN_SECONDS; - -import java.time.Instant; -import java.util.ConcurrentModificationException; -import java.util.concurrent.atomic.AtomicReference; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.ResourceAlreadyExistsException; @@ -21,58 +15,63 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; - import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceState; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelLockService; -import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.DatasourceUpdateService; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.time.Instant; +import java.util.ConcurrentModificationException; +import java.util.concurrent.atomic.AtomicReference; + +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; + /** - * Transport action to create datasource + * Transport action to create tif job */ -public class PutDatasourceTransportAction extends HandledTransportAction { +public class TransportPutTIFJobAction extends HandledTransportAction { private static final Logger log = LogManager.getLogger(DetectorTrigger.class); private final ThreadPool threadPool; - private final DatasourceDao datasourceDao; - private final DatasourceUpdateService datasourceUpdateService; - private final ThreatIntelLockService lockService; + private final TIFJobParameterService tifJobParameterService; + private final TIFJobUpdateService tifJobUpdateService; + private final TIFLockService lockService; /** * Default constructor * @param transportService the transport service * @param actionFilters the action filters * @param threadPool the thread pool - * @param datasourceDao the datasource facade - * @param datasourceUpdateService the datasource update service + * @param tifJobParameterService the tif job parameter service facade + * @param tifJobUpdateService the tif job update service * @param lockService the lock service */ @Inject - public PutDatasourceTransportAction( + public TransportPutTIFJobAction( final TransportService transportService, final ActionFilters actionFilters, final ThreadPool threadPool, - final DatasourceDao datasourceDao, - final DatasourceUpdateService datasourceUpdateService, - final ThreatIntelLockService lockService + final TIFJobParameterService tifJobParameterService, + final TIFJobUpdateService tifJobUpdateService, + final TIFLockService lockService ) { - super(PutDatasourceAction.NAME, transportService, actionFilters, PutDatasourceRequest::new); + super(PutTIFJobAction.NAME, transportService, actionFilters, PutTIFJobRequest::new); this.threadPool = threadPool; - this.datasourceDao = datasourceDao; - this.datasourceUpdateService = datasourceUpdateService; + this.tifJobParameterService = tifJobParameterService; + this.tifJobUpdateService = tifJobUpdateService; this.lockService = lockService; } @Override - protected void doExecute(final Task task, final PutDatasourceRequest request, final ActionListener listener) { + protected void doExecute(final Task task, final PutTIFJobRequest request, final ActionListener listener) { lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { listener.onFailure( @@ -99,15 +98,15 @@ protected void doExecute(final Task task, final PutDatasourceRequest request, fi * unless exception is thrown */ protected void internalDoExecute( - final PutDatasourceRequest request, + final PutTIFJobRequest request, final LockModel lock, final ActionListener listener ) { StepListener createIndexStep = new StepListener<>(); - datasourceDao.createIndexIfNotExists(createIndexStep); + tifJobParameterService.createIndexIfNotExists(createIndexStep); createIndexStep.whenComplete(v -> { - Datasource datasource = Datasource.Builder.build(request); - datasourceDao.putDatasource(datasource, getIndexResponseListener(datasource, lock, listener)); + TIFJobParameter tifJobParameter = TIFJobParameter.Builder.build(request); + tifJobParameterService.putTIFJobParameter(tifJobParameter, getIndexResponseListener(tifJobParameter, lock, listener)); }, exception -> { lockService.releaseLock(lock); log.error("failed to release lock", exception); @@ -120,19 +119,19 @@ protected void internalDoExecute( * unless exception is thrown */ protected ActionListener getIndexResponseListener( - final Datasource datasource, + final TIFJobParameter tifJobParameter, final LockModel lock, final ActionListener listener ) { return new ActionListener<>() { @Override public void onResponse(final IndexResponse indexResponse) { - // This is user initiated request. Therefore, we want to handle the first datasource update task in a generic thread + // This is user initiated request. Therefore, we want to handle the first tifJobParameter update task in a generic thread // pool. threadPool.generic().submit(() -> { AtomicReference lockReference = new AtomicReference<>(lock); try { - createDatasource(datasource, lockService.getRenewLockRunnable(lockReference)); + createTIFJob(tifJobParameter, lockService.getRenewLockRunnable(lockReference)); } finally { lockService.releaseLock(lockReference.get()); } @@ -144,8 +143,8 @@ public void onResponse(final IndexResponse indexResponse) { public void onFailure(final Exception e) { lockService.releaseLock(lock); if (e instanceof VersionConflictEngineException) { - log.error("datasource already exists"); - listener.onFailure(new ResourceAlreadyExistsException("datasource [{}] already exists", datasource.getName())); + log.error("tifJobParameter already exists"); + listener.onFailure(new ResourceAlreadyExistsException("tifJobParameter [{}] already exists", tifJobParameter.getName())); } else { log.error("Internal server error"); listener.onFailure(e); @@ -154,28 +153,28 @@ public void onFailure(final Exception e) { }; } - protected void createDatasource(final Datasource datasource, final Runnable renewLock) { - if (DatasourceState.CREATING.equals(datasource.getState()) == false) { - log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.CREATING, datasource.getState()); - markDatasourceAsCreateFailed(datasource); + protected void createTIFJob(final TIFJobParameter tifJobParameter, final Runnable renewLock) { + if (TIFJobState.CREATING.equals(tifJobParameter.getState()) == false) { + log.error("Invalid tifJobParameter state. Expecting {} but received {}", TIFJobState.CREATING, tifJobParameter.getState()); + markTIFJobAsCreateFailed(tifJobParameter); return; } try { - datasourceUpdateService.updateOrCreateThreatIntelFeedData(datasource, renewLock); + tifJobUpdateService.createThreatIntelFeedData(tifJobParameter, renewLock); } catch (Exception e) { - log.error("Failed to create datasource for {}", datasource.getName(), e); - markDatasourceAsCreateFailed(datasource); + log.error("Failed to create tifJobParameter for {}", tifJobParameter.getName(), e); + markTIFJobAsCreateFailed(tifJobParameter); } } - private void markDatasourceAsCreateFailed(final Datasource datasource) { - datasource.getUpdateStats().setLastFailedAt(Instant.now()); - datasource.setState(DatasourceState.CREATE_FAILED); + private void markTIFJobAsCreateFailed(final TIFJobParameter tifJobParameter) { + tifJobParameter.getUpdateStats().setLastFailedAt(Instant.now()); + tifJobParameter.setState(TIFJobState.CREATE_FAILED); try { - datasourceDao.updateDatasource(datasource); + tifJobParameterService.updateJobSchedulerParameter(tifJobParameter); } catch (Exception e) { - log.error("Failed to mark datasource state as CREATE_FAILED for {}", datasource.getName(), e); + log.error("Failed to mark tifJobParameter state as CREATE_FAILED for {}", tifJobParameter.getName(), e); } } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportUpdateTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportUpdateTIFJobAction.java new file mode 100644 index 000000000..393bc02b9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportUpdateTIFJobAction.java @@ -0,0 +1,133 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobTask; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Locale; + +/** + * Transport action to update tif job + */ +public class TransportUpdateTIFJobAction extends HandledTransportAction { + private static final long LOCK_DURATION_IN_SECONDS = 300l; + private final TIFLockService lockService; + private final TIFJobParameterService tifJobParameterService; + private final TIFJobUpdateService tifJobUpdateService; + private final ThreadPool threadPool; + + /** + * Constructor + * + * @param transportService the transport service + * @param actionFilters the action filters + * @param lockService the lock service + * @param tifJobParameterService the tif job parameter facade + * @param tifJobUpdateService the tif job update service + */ + @Inject + public TransportUpdateTIFJobAction( + final TransportService transportService, + final ActionFilters actionFilters, + final TIFLockService lockService, + final TIFJobParameterService tifJobParameterService, + final TIFJobUpdateService tifJobUpdateService, + final ThreadPool threadPool + ) { + super(UpdateTIFJobAction.NAME, transportService, actionFilters, UpdateTIFJobRequest::new); + this.lockService = lockService; + this.tifJobUpdateService = tifJobUpdateService; + this.tifJobParameterService = tifJobParameterService; + this.threadPool = threadPool; + } + + /** + * Get a lock and update tif job + * + * @param task the task + * @param request the request + * @param listener the listener + */ + @Override + protected void doExecute(final Task task, final UpdateTIFJobRequest request, final ActionListener listener) { + lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + listener.onFailure( + new OpenSearchStatusException("Another processor is holding a lock on the resource. Try again later", RestStatus.BAD_REQUEST) + ); + return; + } + try { + // TODO: makes every sub-methods as async call to avoid using a thread in generic pool + threadPool.generic().submit(() -> { + try { + TIFJobParameter tifJobParameter = tifJobParameterService.getJobParameter(request.getName()); + if (tifJobParameter == null) { + throw new ResourceNotFoundException("no such tifJobParameter exist"); + } + if (TIFJobState.AVAILABLE.equals(tifJobParameter.getState()) == false) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "tif job is not in an [%s] state", TIFJobState.AVAILABLE) + ); + } + updateIfChanged(request, tifJobParameter); //TODO: just want to update? + lockService.releaseLock(lock); + listener.onResponse(new AcknowledgedResponse(true)); + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + } + }); + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + } + }, exception -> listener.onFailure(exception))); + } + + private void updateIfChanged(final UpdateTIFJobRequest request, final TIFJobParameter tifJobParameter) { + boolean isChanged = false; + if (isUpdateIntervalChanged(request)) { + tifJobParameter.setSchedule(new IntervalSchedule(Instant.now(), (int) request.getUpdateInterval().getDays(), ChronoUnit.DAYS)); + tifJobParameter.setTask(TIFJobTask.ALL); + isChanged = true; + } + + if (isChanged) { + tifJobParameterService.updateJobSchedulerParameter(tifJobParameter); + } + } + + /** + * Update interval is changed as long as user provide one because + * start time will get updated even if the update interval is same as current one. + * + * @param request the update tif job request + * @return true if update interval is changed, and false otherwise + */ + private boolean isUpdateIntervalChanged(final UpdateTIFJobRequest request) { + return request.getUpdateInterval() != null; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobAction.java similarity index 54% rename from src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobAction.java index ddf2d42e6..8b4c495f4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobAction.java @@ -9,19 +9,19 @@ import org.opensearch.action.support.master.AcknowledgedResponse; /** - * threat intel datasource update action + * threat intel tif job update action */ -public class UpdateDatasourceAction extends ActionType { +public class UpdateTIFJobAction extends ActionType { /** - * Update datasource action instance + * Update tif job action instance */ - public static final UpdateDatasourceAction INSTANCE = new UpdateDatasourceAction(); + public static final UpdateTIFJobAction INSTANCE = new UpdateTIFJobAction(); /** - * Update datasource action name + * Update tif job action name */ - public static final String NAME = "cluster:admin/security_analytics/datasource/update"; + public static final String NAME = "cluster:admin/security_analytics/tifjob/update"; - private UpdateDatasourceAction() { + private UpdateTIFJobAction() { super(NAME, AcknowledgedResponse::new); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobRequest.java new file mode 100644 index 000000000..205590319 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobRequest.java @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Locale; + +/** + * threat intel tif job update request + */ +public class UpdateTIFJobRequest extends ActionRequest { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); + private static final ParameterValidator VALIDATOR = new ParameterValidator(); + + /** + * @param name the tif job name + * @return the tif job name + */ + private String name; + + /** + * @param updateInterval update interval of a tif job + * @return update interval of a tif job + */ + private TimeValue updateInterval; + + /** + * Parser of a tif job + */ + public static final ObjectParser PARSER; + static { + PARSER = new ObjectParser<>("update_tifjob"); + PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); + } + + public String getName() { + return name; + } + + public TimeValue getUpdateInterval() { + return updateInterval; + } + + private void setUpdateInterval(TimeValue updateInterval){ + this.updateInterval = updateInterval; + } + + /** + * Constructor + * @param name name of a tif job + */ + public UpdateTIFJobRequest(final String name) { + this.name = name; + } + + /** + * Constructor + * @param in the stream input + * @throws IOException IOException + */ + public UpdateTIFJobRequest(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.updateInterval = in.readOptionalTimeValue(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeOptionalTimeValue(updateInterval); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + if (VALIDATOR.validateTIFJobName(name).isEmpty() == false) { + errors.addValidationError("no such tif job exist"); + } + if (updateInterval == null) { + errors.addValidationError("no values to update"); + } + + validateUpdateInterval(errors); + + return errors.validationErrors().isEmpty() ? null : errors; + } + + /** + * Validate updateInterval is equal or larger than 1 + * + * @param errors the errors to add error messages + */ + private void validateUpdateInterval(final ActionRequestValidationException errors) { + if (updateInterval == null) { + return; + } + + if (updateInterval.compareTo(TimeValue.timeValueDays(1)) < 0) { + errors.addValidationError("Update interval should be equal to or larger than 1 day"); + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedMetadata.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedMetadata.java new file mode 100644 index 000000000..7d219a164 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedMetadata.java @@ -0,0 +1,287 @@ +package org.opensearch.securityanalytics.threatIntel.common; + +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +/** + * Database of a tif job + */ +public class FeedMetadata implements Writeable, ToXContent { //feedmetadata + private static final ParseField FEED_ID = new ParseField("feed_id"); + private static final ParseField FEED_NAME = new ParseField("feed_name"); + private static final ParseField FEED_FORMAT = new ParseField("feed_format"); + private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); + private static final ParseField DESCRIPTION = new ParseField("description"); + private static final ParseField ORGANIZATION = new ParseField("organization"); + private static final ParseField CONTAINED_IOCS_FIELD = new ParseField("contained_iocs_field"); + private static final ParseField IOC_COL = new ParseField("ioc_col"); + private static final ParseField FIELDS_FIELD = new ParseField("fields"); + + /** + * @param feedId id of the feed + * @return id of the feed + */ + private String feedId; + + /** + * @param feedFormat format of the feed (csv, json...) + * @return the type of feed ingested + */ + private String feedFormat; + + /** + * @param endpoint URL of a manifest file + * @return URL of a manifest file + */ + private String endpoint; + + /** + * @param feedName name of the threat intel feed + * @return name of the threat intel feed + */ + private String feedName; + + /** + * @param description description of the threat intel feed + * @return description of the threat intel feed + */ + private String description; + + /** + * @param organization organization of the threat intel feed + * @return organization of the threat intel feed + */ + private String organization; + + /** + * @param contained_iocs_field list of iocs contained in a given feed + * @return list of iocs contained in a given feed + */ + private List contained_iocs_field; + + /** + * @param ioc_col column of the contained ioc + * @return column of the contained ioc + */ + private String iocCol; + + /** + * @param fields A list of available fields in the database + * @return A list of available fields in the database + */ + private List fields; + + public FeedMetadata(String feedId, String feedName, String feedFormat, final String endpoint, final String description, + final String organization, final List contained_iocs_field, final String iocCol, final List fields) { + this.feedId = feedId; + this.feedName = feedName; + this.feedFormat = feedFormat; + this.endpoint = endpoint; + this.description = description; + this.organization = organization; + this.contained_iocs_field = contained_iocs_field; + this.iocCol = iocCol; + this.fields = fields; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "tif_metadata_database", + true, + args -> { + String feedId = (String) args[0]; + String feedName = (String) args[1]; + String feedFormat = (String) args[2]; + String endpoint = (String) args[3]; + String description = (String) args[4]; + String organization = (String) args[5]; + List contained_iocs_field = (List) args[6]; + String iocCol = (String) args[7]; + List fields = (List) args[8]; + return new FeedMetadata(feedFormat, endpoint, feedId, feedName, description, organization, contained_iocs_field, iocCol, fields); + } + ); + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FEED_ID); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FEED_NAME); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FEED_FORMAT); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ENDPOINT_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DESCRIPTION); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ORGANIZATION); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), CONTAINED_IOCS_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), IOC_COL); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), FIELDS_FIELD); + } + + public FeedMetadata(final StreamInput in) throws IOException { + feedId = in.readString(); + feedName = in.readString(); + feedFormat = in.readString(); + endpoint = in.readString(); + description = in.readString(); + organization = in.readString(); + contained_iocs_field = in.readStringList(); + iocCol = in.readString(); + fields = in.readOptionalStringList(); + } + + private FeedMetadata(){} + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(feedId); + out.writeString(feedName); + out.writeString(feedFormat); + out.writeString(endpoint); + out.writeString(description); + out.writeString(organization); + out.writeStringCollection(contained_iocs_field); + out.writeString(iocCol); + out.writeOptionalStringCollection(fields); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(FEED_ID.getPreferredName(), feedId); + builder.field(FEED_NAME.getPreferredName(), feedName); + builder.field(FEED_FORMAT.getPreferredName(), feedFormat); + builder.field(ENDPOINT_FIELD.getPreferredName(), endpoint); + builder.field(DESCRIPTION.getPreferredName(), description); + builder.field(ORGANIZATION.getPreferredName(), organization); + builder.field(CONTAINED_IOCS_FIELD.getPreferredName(), contained_iocs_field); + builder.field(IOC_COL.getPreferredName(), iocCol); + +// if (provider != null) { +// builder.field(PROVIDER_FIELD.getPreferredName(), provider); +// } +// if (updatedAt != null) { +// builder.timeField( +// UPDATED_AT_FIELD.getPreferredName(), +// UPDATED_AT_FIELD_READABLE.getPreferredName(), +// updatedAt.toEpochMilli() +// ); +// } + if (fields != null) { + builder.startArray(FIELDS_FIELD.getPreferredName()); + for (String field : fields) { + builder.value(field); + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + + public String getFeedId() { + return feedId; + } + + public String getFeedFormat() { + return feedFormat; + } + + public String getFeedName() { + return feedName; + } + + public String getDescription() { + return description; + } + + public String getOrganization() { + return organization; + } + + public List getContained_iocs_field() { + return contained_iocs_field; + } + + public String getIocCol() { + return iocCol; + } + + public String getEndpoint() { + return this.endpoint; + } + + public List getFields() { + return fields; + } + public void setFeedId(String feedId) { + this.feedId = feedId; + } + + public void setFeedFormat(String feedFormat) { + this.feedFormat = feedFormat; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public void setFeedName(String feedName) { + this.feedName = feedName; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setOrganization(String organization) { + this.organization = organization; + } + + public void setContained_iocs_field(List contained_iocs_field) { + this.contained_iocs_field = contained_iocs_field; + } + + public void setIocCol(String iocCol) { + this.iocCol = iocCol; + } + + public void setFields(List fields) { + this.fields = fields; + } + + /** + * Reset database so that it can be updated in next run regardless there is new update or not + */ + public void resetTIFMetadata() { + this.setFeedId(null); + this.setFeedName(null); + this.setFeedFormat(null); + this.setEndpoint(null); + this.setDescription(null); + this.setOrganization(null); + this.setContained_iocs_field(null); + this.setIocCol(null); + this.setFeedFormat(null); + } + + /** + * Set database attributes with given input + * + * @param tifMetadata the tif metadata + * @param fields the fields + */ + public void setTIFMetadata(final TIFMetadata tifMetadata, final List fields) { + this.feedId = tifMetadata.getFeedId(); + this.feedName = tifMetadata.getName(); + this.feedFormat = tifMetadata.getFeedType(); + this.endpoint = tifMetadata.getUrl(); + this.organization = tifMetadata.getOrganization(); + this.description = tifMetadata.getDescription(); + this.contained_iocs_field = tifMetadata.getContainedIocs(); + this.iocCol = tifMetadata.getIocCol(); + this.fields = fields; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelExecutor.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFExecutor.java similarity index 71% rename from src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelExecutor.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFExecutor.java index b3817786c..c2f861332 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelExecutor.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFExecutor.java @@ -15,16 +15,16 @@ /** * Provide a list of static methods related with executors for threat intel */ -public class ThreatIntelExecutor { - private static final String THREAD_POOL_NAME = "plugin_sap_datasource_update"; +public class TIFExecutor { + private static final String THREAD_POOL_NAME = "_plugin_sap_tifjob_update"; //TODO: name private final ThreadPool threadPool; - public ThreatIntelExecutor(final ThreadPool threadPool) { + public TIFExecutor(final ThreadPool threadPool) { this.threadPool = threadPool; } /** - * We use fixed thread count of 1 for updating datasource as updating datasource is running background + * We use fixed thread count of 1 for updating tif job as updating tif job is running background * once a day at most and no need to expedite the task. * * @param settings the settings @@ -35,11 +35,11 @@ public static ExecutorBuilder executorBuilder(final Settings settings) { } /** - * Return an executor service for datasource update task + * Return an executor service for tif job update task * * @return the executor service */ - public ExecutorService forDatasourceUpdate() { + public ExecutorService forJobSchedulerParameterUpdate() { return threadPool.executor(THREAD_POOL_NAME); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java new file mode 100644 index 000000000..22ffee3e9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +/** + * Threat intel tif job state + * + * When tif job is created, it starts with CREATING state. Once the first threat intel feed is generated, the state changes to AVAILABLE. + * Only when the first threat intel feed generation failed, the state changes to CREATE_FAILED. + * Subsequent threat intel feed failure won't change tif job state from AVAILABLE to CREATE_FAILED. + * When delete request is received, the tif job state changes to DELETING. + * + * State changed from left to right for the entire lifecycle of a datasource + * (CREATING) to (CREATE_FAILED or AVAILABLE) to (DELETING) + * + */ +public enum TIFJobState { + /** + * tif job is being created + */ + CREATING, + /** + * tif job is ready to be used + */ + AVAILABLE, + /** + * tif job creation failed + */ + CREATE_FAILED, + /** + * tif job is being deleted + */ + DELETING +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelLockService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java similarity index 83% rename from src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelLockService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java index 8847d681e..df1fd1b75 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelLockService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java @@ -5,7 +5,7 @@ package org.opensearch.securityanalytics.threatIntel.common; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.DatasourceExtension.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME; import java.time.Instant; import java.util.Optional; @@ -23,11 +23,12 @@ import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.utils.LockService; import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; /** - * A wrapper of job scheduler's lock service for datasource + * A wrapper of job scheduler's lock service */ -public class ThreatIntelLockService { +public class TIFLockService { private static final Logger log = LogManager.getLogger(DetectorTrigger.class); public static final long LOCK_DURATION_IN_SECONDS = 300l; @@ -43,7 +44,7 @@ public class ThreatIntelLockService { * @param clusterService the cluster service * @param client the client */ - public ThreatIntelLockService(final ClusterService clusterService, final Client client) { + public TIFLockService(final ClusterService clusterService, final Client client) { this.clusterService = clusterService; this.lockService = new LockService(client, clusterService); } @@ -51,28 +52,28 @@ public ThreatIntelLockService(final ClusterService clusterService, final Client /** * Wrapper method of LockService#acquireLockWithId * - * Datasource uses its name as doc id in job scheduler. Therefore, we can use datasource name to acquire - * a lock on a datasource. + * tif job uses its name as doc id in job scheduler. Therefore, we can use tif job name to acquire + * a lock on a tif job. * - * @param datasourceName datasourceName to acquire lock on + * @param tifJobName tifJobName to acquire lock on * @param lockDurationSeconds the lock duration in seconds * @param listener the listener */ - public void acquireLock(final String datasourceName, final Long lockDurationSeconds, final ActionListener listener) { - lockService.acquireLockWithId(JOB_INDEX_NAME, lockDurationSeconds, datasourceName, listener); + public void acquireLock(final String tifJobName, final Long lockDurationSeconds, final ActionListener listener) { + lockService.acquireLockWithId(JOB_INDEX_NAME, lockDurationSeconds, tifJobName, listener); } /** * Synchronous method of #acquireLock * - * @param datasourceName datasourceName to acquire lock on + * @param tifJobName tifJobName to acquire lock on * @param lockDurationSeconds the lock duration in seconds * @return lock model */ - public Optional acquireLock(final String datasourceName, final Long lockDurationSeconds) { + public Optional acquireLock(final String tifJobName, final Long lockDurationSeconds) { AtomicReference lockReference = new AtomicReference(); CountDownLatch countDownLatch = new CountDownLatch(1); - lockService.acquireLockWithId(JOB_INDEX_NAME, lockDurationSeconds, datasourceName, new ActionListener<>() { + lockService.acquireLockWithId(JOB_INDEX_NAME, lockDurationSeconds, tifJobName, new ActionListener<>() { @Override public void onResponse(final LockModel lockModel) { lockReference.set(lockModel); @@ -88,7 +89,7 @@ public void onFailure(final Exception e) { }); try { - countDownLatch.await(clusterService.getClusterSettings().get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT).getSeconds(), TimeUnit.SECONDS); + countDownLatch.await(clusterService.getClusterSettings().get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT).getSeconds(), TimeUnit.SECONDS); return Optional.ofNullable(lockReference.get()); } catch (InterruptedException e) { log.error("Waiting for the count down latch failed", e); @@ -133,7 +134,7 @@ public void onFailure(final Exception e) { }); try { - countDownLatch.await(clusterService.getClusterSettings().get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT).getSeconds(), TimeUnit.SECONDS); + countDownLatch.await(clusterService.getClusterSettings().get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT).getSeconds(), TimeUnit.SECONDS); return lockReference.get(); } catch (InterruptedException e) { log.error("Interrupted exception", e); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java new file mode 100644 index 000000000..a594537be --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java @@ -0,0 +1,309 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.common; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLConnection; +import java.nio.CharBuffer; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.SpecialPermission; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.*; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +/** + * Threat intel tif job metadata object + * + * TIFMetadata is stored in an external endpoint. OpenSearch read the file and store values it in this object. + */ +public class TIFMetadata implements Writeable, ToXContent { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + + private static final ParseField FEED_ID = new ParseField("id"); + private static final ParseField URL_FIELD = new ParseField("url"); + private static final ParseField NAME = new ParseField("name"); + private static final ParseField ORGANIZATION = new ParseField("organization"); + private static final ParseField DESCRIPTION = new ParseField("description"); + private static final ParseField FEED_TYPE = new ParseField("feed_type"); + private static final ParseField CONTAINED_IOCS = new ParseField("contained_iocs"); + private static final ParseField IOC_COL = new ParseField("ioc_col"); + + /** + * @param feedId ID of the threat intel feed data + * @return ID of the threat intel feed data + */ + private String feedId; + + /** + * @param url URL of the threat intel feed data + * @return URL of the threat intel feed data + */ + private String url; + + /** + * @param name Name of the threat intel feed + * @return Name of the threat intel feed + */ + private String name; + + /** + * @param organization A threat intel feed organization name + * @return A threat intel feed organization name + */ + private String organization; + + /** + * @param description A description of the database + * @return A description of a database + */ + private String description; + + /** + * @param feedType The type of the data feed (csv, json...) + * @return The type of the data feed (csv, json...) + */ + private String feedType; + + /** + * @param iocCol the column of the ioc data if feedType is csv + * @return the column of the ioc data if feedType is csv + */ + private String iocCol; + + /** + * @param containedIocs list of ioc types contained in feed + * @return list of ioc types contained in feed + */ + private List containedIocs; + + + public String getUrl() { + return url; + } + public String getName() { + return name; + } + public String getOrganization() { + return organization; + } + public String getDescription() { + return description; + } + public String getFeedId() { + return feedId; + } + public String getFeedType() { + return feedType; + } + public String getIocCol() { + return iocCol; + } + public List getContainedIocs() { + return containedIocs; + } + + public void setFeedId(String feedId) { + this.feedId = feedId; + } + + public void setUrl(String url) { + this.url = url; + } + + public void setName(String name) { + this.name = name; + } + + public void setOrganization(String organization) { + this.organization = organization; + } + + public void setFeedType(String feedType) { + this.feedType = feedType; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setIocCol(String iocCol) { + this.iocCol = iocCol; + } + + public void setContainedIocs(List containedIocs) { + this.containedIocs = containedIocs; + } + + + public TIFMetadata(final String feedId, final String url, final String name, final String organization, final String description, + final String feedType, final List containedIocs, final String iocCol) { + this.feedId = feedId; + this.url = url; + this.name = name; + this.organization = organization; + this.description = description; + this.feedType = feedType; + this.containedIocs = containedIocs; + this.iocCol = iocCol; + } + + /** + * tif job metadata parser + */ + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "tif_metadata", + true, + args -> { + String feedId = (String) args[0]; + String url = (String) args[1]; + String name = (String) args[2]; + String organization = (String) args[3]; + String description = (String) args[4]; + String feedType = (String) args[5]; + List containedIocs = (List) args[6]; + String iocCol = (String) args[7]; + return new TIFMetadata(feedId, url, name, organization, description, feedType, containedIocs, iocCol); + } + ); + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), FEED_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), URL_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME); + PARSER.declareString(ConstructingObjectParser.constructorArg(), ORGANIZATION); + PARSER.declareString(ConstructingObjectParser.constructorArg(), DESCRIPTION); + PARSER.declareString(ConstructingObjectParser.constructorArg(), FEED_TYPE); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), CONTAINED_IOCS); + PARSER.declareString(ConstructingObjectParser.constructorArg(), IOC_COL); + } + + public TIFMetadata(final StreamInput in) throws IOException{ + feedId = in.readString(); + url = in.readString(); + name = in.readString(); + organization = in.readString(); + description = in.readString(); + feedType = in.readString(); + containedIocs = in.readStringList(); + iocCol = in.readString(); + } + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(feedId); + out.writeString(url); + out.writeString(name); + out.writeString(organization); + out.writeString(description); + out.writeString(feedType); + out.writeStringCollection(containedIocs); + out.writeString(iocCol); + } + + private TIFMetadata(){} + + + /** + * Reset database so that it can be updated in next run regardless there is new update or not + */ + public void resetTIFMetadata() { + this.setFeedId(null); + this.setUrl(null); + this.setName(null); + this.setOrganization(null); + this.setDescription(null); + this.setFeedType(null); + this.setContainedIocs(null); + this.setIocCol(null); + } + + /** + * Set database attributes with given input + * + * @param tifMetadata the tif metadata + * @param fields the fields + */ + public void setTIFMetadata(final TIFMetadata tifMetadata, final List fields) { + this.feedId = tifMetadata.getFeedId(); + this.url = tifMetadata.getUrl(); + this.name = tifMetadata.getName(); + this.organization = tifMetadata.getOrganization(); + this.description = tifMetadata.getDescription(); + this.feedType = tifMetadata.getFeedType(); + this.containedIocs = tifMetadata.getContainedIocs(); + this.iocCol = tifMetadata.getIocCol(); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(FEED_ID.getPreferredName(), feedId); + builder.field(URL_FIELD.getPreferredName(), url); + builder.field(NAME.getPreferredName(), name); + builder.field(ORGANIZATION.getPreferredName(), organization); + builder.field(DESCRIPTION.getPreferredName(), description); + builder.field(FEED_TYPE.getPreferredName(), feedType); + builder.field(CONTAINED_IOCS.getPreferredName(), containedIocs); + builder.field(IOC_COL.getPreferredName(), iocCol); + builder.endObject(); + return builder; + } + + /** + * TIFMetadata builder + */ + public static class Builder { //TODO: builder? + private static final int FILE_MAX_BYTES = 1024 * 8; + + /** + * Build TIFMetadata from a given url + * + * @param url url to downloads a manifest file + * @return TIFMetadata representing the manifest file + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read manifest file") + public static TIFMetadata build(final URL url) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + URLConnection connection = url.openConnection(); + return internalBuild(connection); + } catch (IOException e) { + log.error("Runtime exception connecting to the manifest file", e); + throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO + } + }); + } + + @SuppressForbidden(reason = "Need to connect to http endpoint to read manifest file") + protected static TIFMetadata internalBuild(final URLConnection connection) throws IOException { + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + InputStreamReader inputStreamReader = new InputStreamReader(connection.getInputStream()); + try (BufferedReader reader = new BufferedReader(inputStreamReader)) { + CharBuffer charBuffer = CharBuffer.allocate(FILE_MAX_BYTES); + reader.read(charBuffer); + charBuffer.flip(); + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + charBuffer.toString() + ); + return PARSER.parse(parser, null); + } + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceExtension.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtension.java similarity index 60% rename from src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceExtension.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtension.java index 4d32973e6..023323253 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceExtension.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtension.java @@ -5,17 +5,16 @@ package org.opensearch.securityanalytics.threatIntel.jobscheduler; -import org.opensearch.jobscheduler.spi.JobSchedulerExtension; import org.opensearch.jobscheduler.spi.ScheduledJobParser; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; import java.util.Map; -public class DatasourceExtension implements JobSchedulerExtension { +public class TIFJobExtension implements org.opensearch.jobscheduler.spi.JobSchedulerExtension { /** - * Job index name for a datasource + * Job index name for a TIF job */ - public static final String JOB_INDEX_NAME = ".scheduler-security_analytics-threatintel-datasource"; //rename this... + public static final String JOB_INDEX_NAME = ".scheduler-sap-threatintel-job"; /** * Job index setting @@ -23,11 +22,11 @@ public class DatasourceExtension implements JobSchedulerExtension { * We want it to be single shard so that job can be run only in a single node by job scheduler. * We want it to expand to all replicas so that querying to this index can be done locally to reduce latency. */ - public static final Map INDEX_SETTING = Map.of("index.number_of_shards", 1, "index.number_of_replicas", "0-all", "index.hidden", true); + public static final Map INDEX_SETTING = Map.of("index.number_of_shards", 1, "index.auto_expand_replicas", "0-all", "index.hidden", true); @Override public String getJobType() { - return "scheduler_security_analytics_threatintel_datasource"; + return "scheduler_sap_threatintel_job"; } @Override @@ -37,11 +36,11 @@ public String getJobIndex() { @Override public ScheduledJobRunner getJobRunner() { - return DatasourceRunner.getJobRunnerInstance(); + return TIFJobRunner.getJobRunnerInstance(); } @Override public ScheduledJobParser getJobParser() { - return (parser, id, jobDocVersion) -> Datasource.PARSER.parse(parser, null); + return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/Datasource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java similarity index 52% rename from src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/Datasource.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java index 00ff1d419..e347e0e60 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java @@ -16,7 +16,6 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.jobscheduler.spi.schedule.Schedule; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; @@ -27,12 +26,11 @@ import static org.opensearch.common.time.DateUtils.toInstant; -import org.opensearch.securityanalytics.threatIntel.action.PutDatasourceRequest; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceManifest; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceState; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelLockService; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -public class Datasource implements Writeable, ScheduledJobParameter { +public class TIFJobParameter implements Writeable, ScheduledJobParameter { /** * Prefix of indices having threatIntel data */ @@ -49,24 +47,14 @@ public class Datasource implements Writeable, ScheduledJobParameter { private static final ParseField ENABLED_TIME_FIELD = new ParseField("enabled_time"); private static final ParseField ENABLED_TIME_FIELD_READABLE = new ParseField("enabled_time_field"); - // need? - private static final ParseField TASK_FIELD = new ParseField("task"); - public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds"; - /** - * Additional fields for datasource + * Additional fields for tif job */ - private static final ParseField FEED_NAME = new ParseField("feed_name"); - private static final ParseField FEED_FORMAT = new ParseField("feed_format"); - private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); - private static final ParseField DESCRIPTION = new ParseField("description"); - private static final ParseField ORGANIZATION = new ParseField("organization"); - private static final ParseField CONTAINED_IOCS_FIELD = new ParseField("contained_iocs_field"); private static final ParseField STATE_FIELD = new ParseField("state"); private static final ParseField CURRENT_INDEX_FIELD = new ParseField("current_index"); private static final ParseField INDICES_FIELD = new ParseField("indices"); - private static final ParseField DATABASE_FIELD = new ParseField("database"); private static final ParseField UPDATE_STATS_FIELD = new ParseField("update_stats"); + private static final ParseField TASK_FIELD = new ParseField("task"); /** @@ -74,14 +62,14 @@ public class Datasource implements Writeable, ScheduledJobParameter { */ /** - * @param name name of a datasource - * @return name of a datasource + * @param name name of a tif job + * @return name of a tif job */ private String name; /** - * @param lastUpdateTime Last update time of a datasource - * @return Last update time of a datasource + * @param lastUpdateTime Last update time of a tif job + * @return Last update time of a tif job */ private Instant lastUpdateTime; /** @@ -100,110 +88,46 @@ public class Datasource implements Writeable, ScheduledJobParameter { */ private IntervalSchedule schedule; - /** - * @param task Task that {@link DatasourceRunner} will execute - * @return Task that {@link DatasourceRunner} will execute - */ - private DatasourceTask task; - - - /** - * Additional variables for datasource - */ - - /** - * @param feedFormat format of the feed (ip, dns...) - * @return the type of feed ingested - */ - private String feedFormat; - - /** - * @param endpoint URL of a manifest file - * @return URL of a manifest file - */ - private String endpoint; - - /** - * @param feedName name of the threat intel feed - * @return name of the threat intel feed - */ - private String feedName; - - /** - * @param description description of the threat intel feed - * @return description of the threat intel feed - */ - private String description; - - /** - * @param organization organization of the threat intel feed - * @return organization of the threat intel feed - */ - private String organization; /** - * @param contained_iocs_field list of iocs contained in a given feed - * @return list of iocs contained in a given feed + * Additional variables for tif job */ - private List contained_iocs_field; /** - * @param state State of a datasource - * @return State of a datasource + * @param state State of a tif job + * @return State of a tif job */ - private DatasourceState state; + private TIFJobState state; /** * @param currentIndex the current index name having threat intel feed data * @return the current index name having threat intel feed data */ private String currentIndex; + /** * @param indices A list of indices having threat intel feed data including currentIndex * @return A list of indices having threat intel feed data including currentIndex */ private List indices; - /** - * @param database threat intel feed database information - * @return threat intel feed database information - */ - private Database database; + /** * @param updateStats threat intel feed database update statistics * @return threat intel feed database update statistics */ private UpdateStats updateStats; - public DatasourceTask getTask() { - return task; - } - - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - public void setLastUpdateTime(Instant lastUpdateTime) { - this.lastUpdateTime = lastUpdateTime; - } - - public void setOrganization(String organization) { - this.organization = organization; - } - - public void setCurrentIndex(String currentIndex) { - this.currentIndex = currentIndex; - } - - public void setTask(DatasourceTask task) { - this.task = task; - } - + /** + * @param task Task that {@link TIFJobRunner} will execute + * @return Task that {@link TIFJobRunner} will execute + */ + private TIFJobTask task; /** - * Datasource parser + * tif job parser */ - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "datasource_metadata", + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "tifjob_metadata", true, args -> { String name = (String) args[0]; @@ -211,35 +135,21 @@ public void setTask(DatasourceTask task) { Instant enabledTime = args[2] == null ? null : Instant.ofEpochMilli((long) args[2]); boolean isEnabled = (boolean) args[3]; IntervalSchedule schedule = (IntervalSchedule) args[4]; - DatasourceTask task = DatasourceTask.valueOf((String) args[6]); - String feedFormat = (String) args[7]; - String endpoint = (String) args[8]; - String feedName = (String) args[9]; - String description = (String) args[10]; - String organization = (String) args[11]; - List contained_iocs_field = (List) args[12]; - DatasourceState state = DatasourceState.valueOf((String) args[13]); - String currentIndex = (String) args[14]; - List indices = (List) args[15]; - Database database = (Database) args[16]; - UpdateStats updateStats = (UpdateStats) args[17]; - Datasource parameter = new Datasource( + TIFJobTask task = TIFJobTask.valueOf((String) args[5]); + TIFJobState state = TIFJobState.valueOf((String) args[6]); + String currentIndex = (String) args[7]; + List indices = (List) args[8]; + UpdateStats updateStats = (UpdateStats) args[9]; + TIFJobParameter parameter = new TIFJobParameter( name, lastUpdateTime, enabledTime, isEnabled, schedule, task, - feedFormat, - endpoint, - feedName, - description, - organization, - contained_iocs_field, state, currentIndex, indices, - database, updateStats ); return parameter; @@ -252,85 +162,56 @@ public void setTask(DatasourceTask task) { PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ScheduleParser.parse(p), SCHEDULE_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), TASK_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), FEED_FORMAT); - PARSER.declareString(ConstructingObjectParser.constructorArg(), ENDPOINT_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), FEED_NAME); - PARSER.declareString(ConstructingObjectParser.constructorArg(), DESCRIPTION); - PARSER.declareString(ConstructingObjectParser.constructorArg(), ORGANIZATION); - PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), CONTAINED_IOCS_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), STATE_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CURRENT_INDEX_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD); - PARSER.declareObject(ConstructingObjectParser.constructorArg(), Database.PARSER, DATABASE_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), UpdateStats.PARSER, UPDATE_STATS_FIELD); } - public Datasource() { - this(null, null, null, null, null, null, null, null); + public TIFJobParameter() { + this(null, null); } - public Datasource(final String name, final Instant lastUpdateTime, final Instant enabledTime, final Boolean isEnabled, - final IntervalSchedule schedule, DatasourceTask task, final String feedFormat, final String endpoint, - final String feedName, final String description, final String organization, final List contained_iocs_field, - final DatasourceState state, final String currentIndex, final List indices, final Database database, final UpdateStats updateStats) { + public TIFJobParameter(final String name, final Instant lastUpdateTime, final Instant enabledTime, final Boolean isEnabled, + final IntervalSchedule schedule, TIFJobTask task, final TIFJobState state, final String currentIndex, + final List indices, final UpdateStats updateStats) { this.name = name; this.lastUpdateTime = lastUpdateTime; this.enabledTime = enabledTime; this.isEnabled = isEnabled; this.schedule = schedule; this.task = task; - this.feedFormat = feedFormat; - this.endpoint = endpoint; - this.feedName = feedName; - this.description = description; - this.organization = organization; - this.contained_iocs_field = contained_iocs_field; this.state = state; this.currentIndex = currentIndex; this.indices = indices; - this.database = database; this.updateStats = updateStats; } - public Datasource(final String name, final IntervalSchedule schedule, final String feedFormat, final String endpoint, final String feedName, final String description, final String organization, final List contained_iocs_field ) { + public TIFJobParameter(final String name, final IntervalSchedule schedule) { this( name, Instant.now().truncatedTo(ChronoUnit.MILLIS), null, false, schedule, - DatasourceTask.ALL, - feedFormat, - endpoint, - feedName, - description, - organization, - contained_iocs_field, - DatasourceState.CREATING, + TIFJobTask.ALL, + TIFJobState.CREATING, null, new ArrayList<>(), - new Database(), new UpdateStats() ); } - public Datasource(final StreamInput in) throws IOException { + public TIFJobParameter(final StreamInput in) throws IOException { name = in.readString(); lastUpdateTime = toInstant(in.readVLong()); enabledTime = toInstant(in.readOptionalVLong()); isEnabled = in.readBoolean(); schedule = new IntervalSchedule(in); - task = DatasourceTask.valueOf(in.readString()); - feedFormat = in.readString(); - endpoint = in.readString(); - feedName = in.readString(); - description = in.readString(); - organization = in.readString(); - contained_iocs_field = in.readStringList(); - state = DatasourceState.valueOf(in.readString()); + task = TIFJobTask.valueOf(in.readString()); + state = TIFJobState.valueOf(in.readString()); currentIndex = in.readOptionalString(); indices = in.readStringList(); - database = new Database(in); updateStats = new UpdateStats(in); } @@ -341,16 +222,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeBoolean(isEnabled); schedule.writeTo(out); out.writeString(task.name()); - out.writeString(feedFormat); - out.writeString(endpoint); - out.writeString(feedName); - out.writeString(description); - out.writeString(organization); - out.writeStringCollection(contained_iocs_field); out.writeString(state.name()); out.writeOptionalString(currentIndex); out.writeStringCollection(indices); - database.writeTo(out); updateStats.writeTo(out); } @@ -373,51 +247,73 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(ENABLED_FIELD.getPreferredName(), isEnabled); builder.field(SCHEDULE_FIELD.getPreferredName(), schedule); builder.field(TASK_FIELD.getPreferredName(), task.name()); - builder.field(FEED_FORMAT.getPreferredName(), feedFormat); - builder.field(ENDPOINT_FIELD.getPreferredName(), endpoint); - builder.field(FEED_NAME.getPreferredName(), feedName); - builder.field(DESCRIPTION.getPreferredName(), description); - builder.field(ORGANIZATION.getPreferredName(), organization); - builder.field(CONTAINED_IOCS_FIELD.getPreferredName(), contained_iocs_field); builder.field(STATE_FIELD.getPreferredName(), state.name()); if (currentIndex != null) { builder.field(CURRENT_INDEX_FIELD.getPreferredName(), currentIndex); } builder.field(INDICES_FIELD.getPreferredName(), indices); - builder.field(DATABASE_FIELD.getPreferredName(), database); builder.field(UPDATE_STATS_FIELD.getPreferredName(), updateStats); builder.endObject(); return builder; } + // getters and setters + public void setName(String name) { + this.name = name; + } + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + + public void setEnabled(boolean enabled) { + isEnabled = enabled; + } + + public void setIndices(List indices) { + this.indices = indices; + } + @Override public String getName() { return this.name; } - @Override public Instant getLastUpdateTime() { return this.lastUpdateTime; } - @Override public Instant getEnabledTime() { return this.enabledTime; } - @Override public IntervalSchedule getSchedule() { return this.schedule; } - @Override public boolean isEnabled() { return this.isEnabled; } + public TIFJobTask getTask() { + return task; + } + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + public void setCurrentIndex(String currentIndex) { + this.currentIndex = currentIndex; + } + + public void setTask(TIFJobTask task) { + this.task = task; + } @Override public Long getLockDurationSeconds() { - return ThreatIntelLockService.LOCK_DURATION_IN_SECONDS; + return TIFLockService.LOCK_DURATION_IN_SECONDS; + } + + public String getCurrentIndex() { + return currentIndex; } /** @@ -440,9 +336,9 @@ public void disable() { } /** - * Current index name of a datasource + * Current index name of a tif job * - * @return Current index name of a datasource + * @return Current index name of a tif job */ public String currentIndexName() { return currentIndex; @@ -453,64 +349,16 @@ public void setSchedule(IntervalSchedule schedule) { } /** - * Reset database so that it can be updated in next run regardless there is new update or not - */ - public void resetDatabase() { - database.setUpdatedAt(null); - database.setSha256Hash(null); - } - - /** - * Index name for a datasource with given suffix + * Index name for a tif job with given suffix * * @param suffix the suffix of a index name - * @return index name for a datasource with given suffix + * @return index name for a tif job with given suffix */ public String newIndexName(final String suffix) { return String.format(Locale.ROOT, "%s.%s.%s", THREAT_INTEL_DATA_INDEX_NAME_PREFIX, name, suffix); } - /** - * Set database attributes with given input - * - * @param datasourceManifest the datasource manifest - * @param fields the fields - */ - public void setDatabase(final DatasourceManifest datasourceManifest, final List fields) { - this.database.setProvider(datasourceManifest.getOrganization()); - this.database.setSha256Hash(datasourceManifest.getSha256Hash()); - this.database.setUpdatedAt(Instant.ofEpochMilli(datasourceManifest.getUpdatedAt())); - this.database.setFields(fields); - } - - /** - * Checks if the database fields are compatible with the given set of fields. - * - * If database fields are null, it is compatible with any input fields - * as it hasn't been generated before. - * - * @param fields The set of input fields to check for compatibility. - * @return true if the database fields are compatible with the given input fields, false otherwise. - */ - public boolean isCompatible(final List fields) { - if (database.fields == null) { - return true; - } - - if (fields.size() < database.fields.size()) { - return false; - } - - Set fieldsSet = new HashSet<>(fields); - for (String field : database.fields) { - if (fieldsSet.contains(field) == false) { - return false; - } - } - return true; - } - - public DatasourceState getState() { + public TIFJobState getState() { return state; } @@ -518,159 +366,17 @@ public List getIndices() { return indices; } - public void setState(DatasourceState previousState) { + public void setState(TIFJobState previousState) { this.state = previousState; } - public String getEndpoint() { - return this.endpoint; - } - - public Database getDatabase() { - return this.database; - } - public UpdateStats getUpdateStats() { return this.updateStats; } - /** - * Database of a datasource - */ - public static class Database implements Writeable, ToXContent { - private static final ParseField PROVIDER_FIELD = new ParseField("provider"); - private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); - private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at_in_epoch_millis"); - private static final ParseField UPDATED_AT_FIELD_READABLE = new ParseField("updated_at"); - private static final ParseField FIELDS_FIELD = new ParseField("fields"); - - /** - * @param provider A database provider name - * @return A database provider name - */ - private String provider; - /** - * @param sha256Hash SHA256 hash value of a database file - * @return SHA256 hash value of a database file - */ - private String sha256Hash; - - /** - * @param updatedAt A date when the database was updated - * @return A date when the database was updated - */ - private Instant updatedAt; - - /** - * @param fields A list of available fields in the database - * @return A list of available fields in the database - */ - private List fields; - - public Database(String provider, String sha256Hash, Instant updatedAt, List fields) { - this.provider = provider; - this.sha256Hash = sha256Hash; - this.updatedAt = updatedAt; - this.fields = fields; - } - - public void setProvider(String provider) { - this.provider = provider; - } - - public void setSha256Hash(String sha256Hash) { - this.sha256Hash = sha256Hash; - } - - public void setUpdatedAt(Instant updatedAt) { - this.updatedAt = updatedAt; - } - - public void setFields(List fields) { - this.fields = fields; - } - - public Instant getUpdatedAt() { - return updatedAt; - } - - public String getSha256Hash() { - return sha256Hash; - } - - public List getFields() { - return fields; - } - - public String getProvider() { - return provider; - } - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "datasource_metadata_database", - true, - args -> { - String provider = (String) args[0]; - String sha256Hash = (String) args[1]; - Instant updatedAt = args[2] == null ? null : Instant.ofEpochMilli((Long) args[2]); - List fields = (List) args[3]; - return new Database(provider, sha256Hash, updatedAt, fields); - } - ); - static { - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHA256_HASH_FIELD); - PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), FIELDS_FIELD); - } - - public Database(final StreamInput in) throws IOException { - provider = in.readOptionalString(); - sha256Hash = in.readOptionalString(); - updatedAt = toInstant(in.readOptionalVLong()); - fields = in.readOptionalStringList(); - } - - private Database(){} - - @Override - public void writeTo(final StreamOutput out) throws IOException { - out.writeOptionalString(provider); - out.writeOptionalString(sha256Hash); - out.writeOptionalVLong(updatedAt == null ? null : updatedAt.toEpochMilli()); - out.writeOptionalStringCollection(fields); - } - - @Override - public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject(); - if (provider != null) { - builder.field(PROVIDER_FIELD.getPreferredName(), provider); - } - if (sha256Hash != null) { - builder.field(SHA256_HASH_FIELD.getPreferredName(), sha256Hash); - } - if (updatedAt != null) { - builder.timeField( - UPDATED_AT_FIELD.getPreferredName(), - UPDATED_AT_FIELD_READABLE.getPreferredName(), - updatedAt.toEpochMilli() - ); - } - if (fields != null) { - builder.startArray(FIELDS_FIELD.getPreferredName()); - for (String field : fields) { - builder.value(field); - } - builder.endArray(); - } - builder.endObject(); - return builder; - } - } /** - * Update stats of a datasource + * Update stats of a tif job */ public static class UpdateStats implements Writeable, ToXContent { private static final ParseField LAST_SUCCEEDED_AT_FIELD = new ParseField("last_succeeded_at_in_epoch_millis"); @@ -681,6 +387,22 @@ public static class UpdateStats implements Writeable, ToXContent { private static final ParseField LAST_SKIPPED_AT = new ParseField("last_skipped_at_in_epoch_millis"); private static final ParseField LAST_SKIPPED_AT_READABLE = new ParseField("last_skipped_at"); + public Instant getLastSucceededAt() { + return lastSucceededAt; + } + + public Long getLastProcessingTimeInMillis() { + return lastProcessingTimeInMillis; + } + + public Instant getLastFailedAt() { + return lastFailedAt; + } + + public Instant getLastSkippedAt() { + return lastSkippedAt; + } + /** * @param lastSucceededAt The last time when threat intel feed data update was succeeded * @return The last time when threat intel feed data update was succeeded @@ -718,7 +440,7 @@ public void setLastProcessingTimeInMillis(Long lastProcessingTimeInMillis) { } private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "datasource_metadata_update_stats", + "tifjob_metadata_update_stats", true, args -> { Instant lastSucceededAt = args[0] == null ? null : Instant.ofEpochMilli((long) args[0]); @@ -728,7 +450,6 @@ public void setLastProcessingTimeInMillis(Long lastProcessingTimeInMillis) { return new UpdateStats(lastSucceededAt, lastProcessingTimeInMillis, lastFailedAt, lastSkippedAt); } ); - static { PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LAST_SUCCEEDED_AT_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LAST_PROCESSING_TIME_IN_MILLIS_FIELD); @@ -750,7 +471,6 @@ public UpdateStats(Instant lastSucceededAt, Long lastProcessingTimeInMillis, Ins this.lastSkippedAt = lastSkippedAt; } - @Override public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalVLong(lastSucceededAt == null ? null : lastSucceededAt.toEpochMilli()); @@ -795,25 +515,19 @@ public void setLastFailedAt(Instant now) { } } - /** - * Builder class for Datasource + * Builder class for tif job */ public static class Builder { - public static Datasource build(final PutDatasourceRequest request) { - String id = request.getName(); + public static TIFJobParameter build(final PutTIFJobRequest request) { + String name = request.getName(); IntervalSchedule schedule = new IntervalSchedule( Instant.now().truncatedTo(ChronoUnit.MILLIS), (int) request.getUpdateInterval().days(), ChronoUnit.DAYS ); - String feedFormat = request.getFeedFormat(); - String endpoint = request.getEndpoint(); - String feedName = request.getFeedName(); - String description = request.getDescription(); - String organization = request.getOrganization(); - List contained_iocs_field = request.getContained_iocs_field(); - return new Datasource(id, schedule, feedFormat, endpoint, feedName, description, organization, contained_iocs_field); + return new TIFJobParameter(name, schedule); + } } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/dao/DatasourceDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java similarity index 62% rename from src/main/java/org/opensearch/securityanalytics/threatintel/dao/DatasourceDao.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java index 9d6a15241..cab8dcc0b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/dao/DatasourceDao.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.dao; +package org.opensearch.securityanalytics.threatIntel.jobscheduler; import java.io.BufferedReader; import java.io.IOException; @@ -50,9 +50,7 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelSettings; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.DatasourceExtension; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilders; @@ -60,9 +58,9 @@ import org.opensearch.securityanalytics.util.SecurityAnalyticsException; /** - * Data access object for datasource + * Data access object for tif job */ -public class DatasourceDao { +public class TIFJobParameterService { private static final Logger log = LogManager.getLogger(DetectorTrigger.class); private static final Integer MAX_SIZE = 1000; @@ -70,24 +68,24 @@ public class DatasourceDao { private final ClusterService clusterService; private final ClusterSettings clusterSettings; - public DatasourceDao(final Client client, final ClusterService clusterService) { + public TIFJobParameterService(final Client client, final ClusterService clusterService) { this.client = client; this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); } /** - * Create datasource index + * Create tif job index * * @param stepListener setup listener */ public void createIndexIfNotExists(final StepListener stepListener) { - if (clusterService.state().metadata().hasIndex(DatasourceExtension.JOB_INDEX_NAME) == true) { + if (clusterService.state().metadata().hasIndex(TIFJobExtension.JOB_INDEX_NAME) == true) { stepListener.onResponse(null); return; } - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(DatasourceExtension.JOB_INDEX_NAME).mapping(getIndexMapping()) - .settings(DatasourceExtension.INDEX_SETTING); + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(TIFJobExtension.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(TIFJobExtension.INDEX_SETTING); StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { @Override public void onResponse(final CreateIndexResponse createIndexResponse) { @@ -97,7 +95,7 @@ public void onResponse(final CreateIndexResponse createIndexResponse) { @Override public void onFailure(final Exception e) { if (e instanceof ResourceAlreadyExistsException) { - log.info("index[{}] already exist", DatasourceExtension.JOB_INDEX_NAME); + log.info("index[{}] already exist", TIFJobExtension.JOB_INDEX_NAME); stepListener.onResponse(null); return; } @@ -108,7 +106,7 @@ public void onFailure(final Exception e) { private String getIndexMapping() { try { - try (InputStream is = DatasourceDao.class.getResourceAsStream("/mappings/threatintel_datasource.json")) { + try (InputStream is = TIFJobParameterService.class.getResourceAsStream("/mappings/threat_intel_job_mapping.json")) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { return reader.lines().map(String::trim).collect(Collectors.joining()); } @@ -120,21 +118,21 @@ private String getIndexMapping() { } /** - * Update datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param datasource the datasource + * Update jobSchedulerParameter in an index {@code TIFJobExtension.JOB_INDEX_NAME} + * @param jobSchedulerParameter the jobSchedulerParameter * @return index response */ - public IndexResponse updateDatasource(final Datasource datasource) { - datasource.setLastUpdateTime(Instant.now()); + public IndexResponse updateJobSchedulerParameter(final TIFJobParameter jobSchedulerParameter) { + jobSchedulerParameter.setLastUpdateTime(Instant.now()); return StashedThreadContext.run(client, () -> { try { - return client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) - .setId(datasource.getName()) + return client.prepareIndex(TIFJobExtension.JOB_INDEX_NAME) + .setId(jobSchedulerParameter.getName()) .setOpType(DocWriteRequest.OpType.INDEX) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .setSource(jobSchedulerParameter.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) .execute() - .actionGet(clusterSettings.get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT)); + .actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)); } catch (IOException e) { throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO } @@ -142,27 +140,26 @@ public IndexResponse updateDatasource(final Datasource datasource) { } /** - * Update datasources in an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param datasources the datasources + * Update tif jobs in an index {@code TIFJobExtension.JOB_INDEX_NAME} + * @param tifJobParameters the tifJobParameters * @param listener action listener */ - public void updateDatasource(final List datasources, final ActionListener listener) { + public void updateJobSchedulerParameter(final List tifJobParameters, final ActionListener listener) { BulkRequest bulkRequest = new BulkRequest(); - datasources.stream().map(datasource -> { - datasource.setLastUpdateTime(Instant.now()); - return datasource; + tifJobParameters.stream().map(tifJobParameter -> { + tifJobParameter.setLastUpdateTime(Instant.now()); + return tifJobParameter; }).map(this::toIndexRequest).forEach(indexRequest -> bulkRequest.add(indexRequest)); StashedThreadContext.run(client, () -> client.bulk(bulkRequest, listener)); } - - private IndexRequest toIndexRequest(Datasource datasource) { + private IndexRequest toIndexRequest(TIFJobParameter tifJobParameter) { try { IndexRequest indexRequest = new IndexRequest(); - indexRequest.index(DatasourceExtension.JOB_INDEX_NAME); - indexRequest.id(datasource.getName()); + indexRequest.index(TIFJobExtension.JOB_INDEX_NAME); + indexRequest.id(tifJobParameter.getName()); indexRequest.opType(DocWriteRequest.OpType.INDEX); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - indexRequest.source(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); + indexRequest.source(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); return indexRequest; } catch (IOException e) { throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO @@ -170,20 +167,48 @@ private IndexRequest toIndexRequest(Datasource datasource) { } /** - * Put datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} + * Get tif job from an index {@code TIFJobExtension.JOB_INDEX_NAME} + * @param name the name of a tif job + * @return tif job + * @throws IOException exception + */ + public TIFJobParameter getJobParameter(final String name) throws IOException { + GetRequest request = new GetRequest(TIFJobExtension.JOB_INDEX_NAME, name); + GetResponse response; + try { + response = StashedThreadContext.run(client, () -> client.get(request).actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT))); + if (response.isExists() == false) { + log.error("TIF job[{}] does not exist in an index[{}]", name, TIFJobExtension.JOB_INDEX_NAME); + return null; + } + } catch (IndexNotFoundException e) { + log.error("Index[{}] is not found", TIFJobExtension.JOB_INDEX_NAME); + return null; + } + + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + response.getSourceAsBytesRef() + ); + return TIFJobParameter.PARSER.parse(parser, null); + } + + /** + * Put tifJobParameter in an index {@code TIFJobExtension.JOB_INDEX_NAME} * - * @param datasource the datasource + * @param tifJobParameter the tifJobParameter * @param listener the listener */ - public void putDatasource(final Datasource datasource, final ActionListener listener) { - datasource.setLastUpdateTime(Instant.now()); + public void putTIFJobParameter(final TIFJobParameter tifJobParameter, final ActionListener listener) { + tifJobParameter.setLastUpdateTime(Instant.now()); StashedThreadContext.run(client, () -> { try { - client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) - .setId(datasource.getName()) + client.prepareIndex(TIFJobExtension.JOB_INDEX_NAME) + .setId(tifJobParameter.getName()) .setOpType(DocWriteRequest.OpType.CREATE) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .setSource(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) .execute(listener); } catch (IOException e) { throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO @@ -192,63 +217,35 @@ public void putDatasource(final Datasource datasource, final ActionListener list } /** - * Delete datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} + * Delete tifJobParameter in an index {@code TIFJobExtension.JOB_INDEX_NAME} * - * @param datasource the datasource + * @param tifJobParameter the tifJobParameter * */ - public void deleteDatasource(final Datasource datasource) { + public void deleteTIFJobParameter(final TIFJobParameter tifJobParameter) { DeleteResponse response = client.prepareDelete() - .setIndex(DatasourceExtension.JOB_INDEX_NAME) - .setId(datasource.getName()) + .setIndex(TIFJobExtension.JOB_INDEX_NAME) + .setId(tifJobParameter.getName()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .execute() - .actionGet(clusterSettings.get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT)); + .actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)); if (response.status().equals(RestStatus.OK)) { - log.info("deleted datasource[{}] successfully", datasource.getName()); + log.info("deleted tifJobParameter[{}] successfully", tifJobParameter.getName()); } else if (response.status().equals(RestStatus.NOT_FOUND)) { - throw new ResourceNotFoundException("datasource[{}] does not exist", datasource.getName()); + throw new ResourceNotFoundException("tifJobParameter[{}] does not exist", tifJobParameter.getName()); } else { - throw new OpenSearchException("failed to delete datasource[{}] with status[{}]", datasource.getName(), response.status()); + throw new OpenSearchException("failed to delete tifJobParameter[{}] with status[{}]", tifJobParameter.getName(), response.status()); } } /** - * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param name the name of a datasource - * @return datasource - * @throws IOException exception - */ - public Datasource getDatasource(final String name) throws IOException { - GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); - GetResponse response; - try { - response = StashedThreadContext.run(client, () -> client.get(request).actionGet(clusterSettings.get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT))); - if (response.isExists() == false) { - log.error("Datasource[{}] does not exist in an index[{}]", name, DatasourceExtension.JOB_INDEX_NAME); - return null; - } - } catch (IndexNotFoundException e) { - log.error("Index[{}] is not found", DatasourceExtension.JOB_INDEX_NAME); - return null; - } - - XContentParser parser = XContentHelper.createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - response.getSourceAsBytesRef() - ); - return Datasource.PARSER.parse(parser, null); - } - - /** - * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param name the name of a datasource + * Get tif job from an index {@code TIFJobExtension.JOB_INDEX_NAME} + * @param name the name of a tif job * @param actionListener the action listener */ - public void getDatasource(final String name, final ActionListener actionListener) { - GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); + public void getJobParameter(final String name, final ActionListener actionListener) { + GetRequest request = new GetRequest(TIFJobExtension.JOB_INDEX_NAME, name); StashedThreadContext.run(client, () -> client.get(request, new ActionListener<>() { @Override public void onResponse(final GetResponse response) { @@ -263,7 +260,7 @@ public void onResponse(final GetResponse response) { LoggingDeprecationHandler.INSTANCE, response.getSourceAsBytesRef() ); - actionListener.onResponse(Datasource.PARSER.parse(parser, null)); + actionListener.onResponse(TIFJobParameter.PARSER.parse(parser, null)); } catch (IOException e) { actionListener.onFailure(e); } @@ -277,65 +274,65 @@ public void onFailure(final Exception e) { } /** - * Get datasources from an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param names the array of datasource names + * Get tif jobs from an index {@code TIFJobExtension.JOB_INDEX_NAME} + * @param names the array of tif job names * @param actionListener the action listener */ - public void getDatasources(final String[] names, final ActionListener> actionListener) { + public void getTIFJobParameters(final String[] names, final ActionListener> actionListener) { StashedThreadContext.run( client, () -> client.prepareMultiGet() - .add(DatasourceExtension.JOB_INDEX_NAME, names) - .execute(createGetDataSourceQueryActionLister(MultiGetResponse.class, actionListener)) + .add(TIFJobExtension.JOB_INDEX_NAME, names) + .execute(createGetTIFJobParameterQueryActionLister(MultiGetResponse.class, actionListener)) ); } /** - * Get all datasources up to {@code MAX_SIZE} from an index {@code DatasourceExtension.JOB_INDEX_NAME} + * Get all tif jobs up to {@code MAX_SIZE} from an index {@code TIFJobExtension.JOB_INDEX_NAME} * @param actionListener the action listener */ - public void getAllDatasources(final ActionListener> actionListener) { + public void getAllTIFJobParameters(final ActionListener> actionListener) { StashedThreadContext.run( client, - () -> client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) + () -> client.prepareSearch(TIFJobExtension.JOB_INDEX_NAME) .setQuery(QueryBuilders.matchAllQuery()) .setPreference(Preference.PRIMARY.type()) .setSize(MAX_SIZE) - .execute(createGetDataSourceQueryActionLister(SearchResponse.class, actionListener)) + .execute(createGetTIFJobParameterQueryActionLister(SearchResponse.class, actionListener)) ); } /** - * Get all datasources up to {@code MAX_SIZE} from an index {@code DatasourceExtension.JOB_INDEX_NAME} + * Get all tif jobs up to {@code MAX_SIZE} from an index {@code TIFJobExtension.JOB_INDEX_NAME} */ - public List getAllDatasources() { + public List getAllTIFJobParameters() { SearchResponse response = StashedThreadContext.run( client, - () -> client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) + () -> client.prepareSearch(TIFJobExtension.JOB_INDEX_NAME) .setQuery(QueryBuilders.matchAllQuery()) .setPreference(Preference.PRIMARY.type()) .setSize(MAX_SIZE) .execute() - .actionGet(clusterSettings.get(ThreatIntelSettings.THREAT_INTEL_TIMEOUT)) + .actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)) ); List bytesReferences = toBytesReferences(response); - return bytesReferences.stream().map(bytesRef -> toDatasource(bytesRef)).collect(Collectors.toList()); + return bytesReferences.stream().map(bytesRef -> toTIFJobParameter(bytesRef)).collect(Collectors.toList()); } - private ActionListener createGetDataSourceQueryActionLister( + private ActionListener createGetTIFJobParameterQueryActionLister( final Class response, - final ActionListener> actionListener + final ActionListener> actionListener ) { return new ActionListener() { @Override public void onResponse(final T response) { try { List bytesReferences = toBytesReferences(response); - List datasources = bytesReferences.stream() - .map(bytesRef -> toDatasource(bytesRef)) + List tifJobParameters = bytesReferences.stream() + .map(bytesRef -> toTIFJobParameter(bytesRef)) .collect(Collectors.toList()); - actionListener.onResponse(datasources); + actionListener.onResponse(tifJobParameters); } catch (Exception e) { actionListener.onFailure(e); } @@ -365,14 +362,14 @@ private List toBytesReferences(final Object response) { } } - private Datasource toDatasource(final BytesReference bytesReference) { + private TIFJobParameter toTIFJobParameter(final BytesReference bytesReference) { try { XContentParser parser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, bytesReference ); - return Datasource.PARSER.parse(parser, null); + return TIFJobParameter.PARSER.parse(parser, null); } catch (IOException e) { throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java new file mode 100644 index 000000000..dfe16f4c6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -0,0 +1,167 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.securityanalytics.model.DetectorTrigger; + +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.time.Instant; + +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFExecutor; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.threadpool.ThreadPool; + +/** + * Job Parameter update task + * + * This is a background task which is responsible for updating threat intel feed data + */ +public class TIFJobRunner implements ScheduledJobRunner { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + private static TIFJobRunner INSTANCE; + + public static TIFJobRunner getJobRunnerInstance() { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (TIFJobRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new TIFJobRunner(); + return INSTANCE; + } + } + + private ClusterService clusterService; + + // threat intel specific variables + private TIFJobUpdateService jobSchedulerUpdateService; + private TIFJobParameterService jobSchedulerParameterService; + private TIFExecutor threatIntelExecutor; + private TIFLockService lockService; + private boolean initialized; + private ThreadPool threadPool; + + public void setThreadPool(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + private TIFJobRunner() { + // Singleton class, use getJobRunner method instead of constructor + } + + public void initialize( + final ClusterService clusterService, + final TIFJobUpdateService jobSchedulerUpdateService, + final TIFJobParameterService jobSchedulerParameterService, + final TIFExecutor threatIntelExecutor, + final TIFLockService threatIntelLockService, + final ThreadPool threadPool + ) { + this.clusterService = clusterService; + this.jobSchedulerUpdateService = jobSchedulerUpdateService; + this.jobSchedulerParameterService = jobSchedulerParameterService; + this.threatIntelExecutor = threatIntelExecutor; + this.lockService = threatIntelLockService; + this.threadPool = threadPool; + this.initialized = true; + } + + @Override + public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { + if (initialized == false) { + throw new AssertionError("This instance is not initialized"); + } + + log.info("Update job started for a job parameter[{}]", jobParameter.getName()); + if (jobParameter instanceof TIFJobParameter == false) { + log.error("Illegal state exception: job parameter is not instance of Job Scheduler Parameter"); + throw new IllegalStateException( + "job parameter is not instance of Job Scheduler Parameter, type: " + jobParameter.getClass().getCanonicalName() + ); + } + threadPool.generic().submit(updateJobRunner(jobParameter)); +// threatIntelExecutor.forJobSchedulerParameterUpdate().submit(updateJobRunner(jobParameter)); + } + + /** + * Update threat intel feed data + * + * Lock is used so that only one of nodes run this task. + * + * @param jobParameter job parameter + */ + protected Runnable updateJobRunner(final ScheduledJobParameter jobParameter) { + return () -> { + Optional lockModel = lockService.acquireLock( + jobParameter.getName(), + TIFLockService.LOCK_DURATION_IN_SECONDS + ); + if (lockModel.isEmpty()) { + log.error("Failed to update. Another processor is holding a lock for job parameter[{}]", jobParameter.getName()); + return; + } + + LockModel lock = lockModel.get(); + try { + updateJobParameter(jobParameter, lockService.getRenewLockRunnable(new AtomicReference<>(lock))); + } catch (Exception e) { + log.error("Failed to update job parameter[{}]", jobParameter.getName(), e); + } finally { + lockService.releaseLock(lock); + } + }; + } + + protected void updateJobParameter(final ScheduledJobParameter jobParameter, final Runnable renewLock) throws IOException { + TIFJobParameter jobSchedulerParameter = jobSchedulerParameterService.getJobParameter(jobParameter.getName()); + /** + * If delete request comes while update task is waiting on a queue for other update tasks to complete, + * because update task for this jobSchedulerParameter didn't acquire a lock yet, delete request is processed. + * When it is this jobSchedulerParameter's turn to run, it will find that the jobSchedulerParameter is deleted already. + * Therefore, we stop the update process when data source does not exist. + */ + if (jobSchedulerParameter == null) { + log.info("Job parameter[{}] does not exist", jobParameter.getName()); + return; + } + + if (TIFJobState.AVAILABLE.equals(jobSchedulerParameter.getState()) == false) { + log.error("Invalid jobSchedulerParameter state. Expecting {} but received {}", TIFJobState.AVAILABLE, jobSchedulerParameter.getState()); + jobSchedulerParameter.disable(); + jobSchedulerParameter.getUpdateStats().setLastFailedAt(Instant.now()); + jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); + return; + } + try { + jobSchedulerUpdateService.deleteAllTifdIndices(jobSchedulerParameter); + if (TIFJobTask.DELETE_UNUSED_INDICES.equals(jobSchedulerParameter.getTask()) == false) { + jobSchedulerUpdateService.createThreatIntelFeedData(jobSchedulerParameter, renewLock); + } +// jobSchedulerUpdateService.deleteUnusedIndices(jobSchedulerParameter); + } catch (Exception e) { + log.error("Failed to update jobSchedulerParameter for {}", jobSchedulerParameter.getName(), e); + jobSchedulerParameter.getUpdateStats().setLastFailedAt(Instant.now()); + jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); + } finally { +// jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); + jobSchedulerUpdateService.updateJobSchedulerParameter(jobSchedulerParameter, jobSchedulerParameter.getSchedule(), TIFJobTask.ALL); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceTask.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobTask.java similarity index 78% rename from src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceTask.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobTask.java index b0e9ac184..1221a3540 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceTask.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobTask.java @@ -6,9 +6,9 @@ package org.opensearch.securityanalytics.threatIntel.jobscheduler; /** - * Task that {@link DatasourceRunner} will run + * Task that {@link TIFJobRunner} will run */ -public enum DatasourceTask { +public enum TIFJobTask { /** * Do everything */ diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java new file mode 100644 index 000000000..710d8015c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java @@ -0,0 +1,287 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.opensearch.OpenSearchException; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; + +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedParser; +import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +public class TIFJobUpdateService { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + + private static final int SLEEP_TIME_IN_MILLIS = 5000; // 5 seconds + private static final int MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS = 10 * 60 * 60 * 1000; // 10 hours + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final TIFJobParameterService jobSchedulerParameterService; + private final ThreatIntelFeedDataService threatIntelFeedDataService; + + public TIFJobUpdateService( + final ClusterService clusterService, + final TIFJobParameterService jobSchedulerParameterService, + final ThreatIntelFeedDataService threatIntelFeedDataService + ) { + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.jobSchedulerParameterService = jobSchedulerParameterService; + this.threatIntelFeedDataService = threatIntelFeedDataService; + } + + // functions used in job Runner + /** + * Delete all indices except the one which is being used + * + * @param jobSchedulerParameter + */ + public void deleteAllTifdIndices(final TIFJobParameter jobSchedulerParameter) { + try { + List indicesToDelete = jobSchedulerParameter.getIndices() + .stream() +// .filter(index -> index.equals(jobSchedulerParameter.currentIndexName()) == false) + .collect(Collectors.toList()); + + List deletedIndices = deleteIndices(indicesToDelete); + + if (deletedIndices.isEmpty() == false) { + jobSchedulerParameter.getIndices().removeAll(deletedIndices); + jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); + } + } catch (Exception e) { + log.error("Failed to delete old indices for {}", jobSchedulerParameter.getName(), e); + } + } + + /** + * Update jobSchedulerParameter with given systemSchedule and task + * + * @param jobSchedulerParameter jobSchedulerParameter to update + * @param systemSchedule new system schedule value + * @param task new task value + */ + public void updateJobSchedulerParameter(final TIFJobParameter jobSchedulerParameter, final IntervalSchedule systemSchedule, final TIFJobTask task) { + boolean updated = false; + if (jobSchedulerParameter.getSchedule().equals(systemSchedule) == false) { //TODO: will always be true + jobSchedulerParameter.setSchedule(systemSchedule); + updated = true; + } + if (jobSchedulerParameter.getTask().equals(task) == false) { + jobSchedulerParameter.setTask(task); + updated = true; + } // this is called when task == DELETE + if (updated) { + jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); + } + } + + private List deleteIndices(final List indicesToDelete) { + List deletedIndices = new ArrayList<>(indicesToDelete.size()); + for (String index : indicesToDelete) { + if (clusterService.state().metadata().hasIndex(index) == false) { + deletedIndices.add(index); + continue; + } + try { + threatIntelFeedDataService.deleteThreatIntelDataIndex(index); + deletedIndices.add(index); + } catch (Exception e) { + log.error("Failed to delete an index [{}]", index, e); + } + } + return deletedIndices; + } + + + /** + * Update threat intel feed data + * + * The first column is ip range field regardless its header name. + * Therefore, we don't store the first column's header name. + * + * @param jobSchedulerParameter the jobSchedulerParameter + * @param renewLock runnable to renew lock + * + * @throws IOException + */ + public void createThreatIntelFeedData(final TIFJobParameter jobSchedulerParameter, final Runnable renewLock) throws IOException { + // parse YAML containing list of threat intel feeds + // for each feed (ex. Feodo) + // parse feed specific YAML containing TIFMetadata + + // for every threat intel feed + // create and store a new TIFMetadata object + + // use the TIFMetadata to switch case feed type + // parse through file and save threat intel feed data + + List containedIocs = new ArrayList<>(); + TIFMetadata tifMetadata = new TIFMetadata("feedid", "url", "name", "org", + "descr", "csv", containedIocs, "1"); // TODO: example tif metdata + + Instant startTime = Instant.now(); + String indexName = setupIndex(jobSchedulerParameter); + String[] header; + + Boolean succeeded; + + switch(tifMetadata.getFeedType()) { + case "csv": + try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)) { + // iterate until we find first line without '#' + CSVRecord findHeader = reader.iterator().next(); + while (findHeader.get(0).charAt(0) == '#' || findHeader.get(0).charAt(0) == ' ') { + findHeader = reader.iterator().next(); + } + CSVRecord headerLine = findHeader; + header = ThreatIntelFeedParser.validateHeader(headerLine).values(); + + threatIntelFeedDataService.saveThreatIntelFeedDataCSV(indexName, header, reader.iterator(), renewLock, tifMetadata); + } + default: + // if the feed type doesn't match any of the supporting feed types, throw an exception + succeeded = false; + } + + if (!succeeded) { + log.error("Exception: failed to parse correct feed type"); + throw new OpenSearchException("Exception: failed to parse correct feed type"); + } + + // end the loop here + + waitUntilAllShardsStarted(indexName, MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS); + Instant endTime = Instant.now(); + updateJobSchedulerParameterAsSucceeded(indexName, jobSchedulerParameter, startTime, endTime); + } + + // helper functions + /*** + * Update jobSchedulerParameter as succeeded + * + * @param jobSchedulerParameter the jobSchedulerParameter + */ + private void updateJobSchedulerParameterAsSucceeded( + final String newIndexName, + final TIFJobParameter jobSchedulerParameter, + final Instant startTime, + final Instant endTime + ) { + jobSchedulerParameter.setCurrentIndex(newIndexName); // TODO: remove current index? + jobSchedulerParameter.getUpdateStats().setLastSucceededAt(endTime); + jobSchedulerParameter.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); + jobSchedulerParameter.enable(); + jobSchedulerParameter.setState(TIFJobState.AVAILABLE); + jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); + log.info( + "threat intel feed database creation succeeded for {} and took {} seconds", + jobSchedulerParameter.getName(), + Duration.between(startTime, endTime) + ); + } + + /*** + * Setup index to add a new threat intel feed data + * + * @param jobSchedulerParameter the jobSchedulerParameter + * @return new index name + */ + private String setupIndex(final TIFJobParameter jobSchedulerParameter) { + String indexName = jobSchedulerParameter.newIndexName(UUID.randomUUID().toString()); + jobSchedulerParameter.getIndices().add(indexName); + jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); + threatIntelFeedDataService.createIndexIfNotExists(indexName); + return indexName; + } + + /** + * We wait until all shards are ready to serve search requests before updating job scheduler parameter to + * point to a new index so that there won't be latency degradation during threat intel feed data update + * + * @param indexName the indexName + */ + protected void waitUntilAllShardsStarted(final String indexName, final int timeout) { + Instant start = Instant.now(); + try { + while (Instant.now().toEpochMilli() - start.toEpochMilli() < timeout) { + if (clusterService.state().routingTable().allShards(indexName).stream().allMatch(shard -> shard.started())) { + return; + } + Thread.sleep(SLEEP_TIME_IN_MILLIS); + } + throw new OpenSearchException( + "index[{}] replication did not complete after {} millis", + MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS + ); + } catch (InterruptedException e) { + log.error("runtime exception", e); + throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO + } + } + + +// /** +// * Determine if update is needed or not +// * +// * Update is needed when all following conditions are met +// * 1. updatedAt value in jobSchedulerParameter is equal or before updateAt value in tifMetadata +// * 2. SHA256 hash value in jobSchedulerParameter is different with SHA256 hash value in tifMetadata +// * +// * @param jobSchedulerParameter +// * @param tifMetadata +// * @return +// */ +// private boolean shouldUpdate(final TIFJobParameter jobSchedulerParameter, final TIFMetadata tifMetadata) { +// if (jobSchedulerParameter.getDatabase().getUpdatedAt() != null +// && jobSchedulerParameter.getDatabase().getUpdatedAt().toEpochMilli() > tifMetadata.getUpdatedAt()) { +// return false; +// } +// +// if (tifMetadata.getSha256Hash().equals(jobSchedulerParameter.getDatabase().getSha256Hash())) { +// return false; +// } +// return true; +// } + +// /** +// * Return header fields of threat intel feed data with given url of a manifest file +// * +// * The first column is ip range field regardless its header name. +// * Therefore, we don't store the first column's header name. +// * +// * @param TIFMetadataUrl the url of a manifest file +// * @return header fields of threat intel feed +// */ +// public List getHeaderFields(String TIFMetadataUrl) throws IOException { +// URL url = new URL(TIFMetadataUrl); +// TIFMetadata tifMetadata = TIFMetadata.Builder.build(url); +// +// try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)) { +// String[] fields = reader.iterator().next().values(); +// return Arrays.asList(fields).subList(1, fields.length); +// } +// } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceAction.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceAction.java deleted file mode 100644 index 6befdde04..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceAction.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.action.ActionType; - -/** - * Threat intel datasource get action - */ -public class GetDatasourceAction extends ActionType { - /** - * Get datasource action instance - */ - public static final GetDatasourceAction INSTANCE = new GetDatasourceAction(); - /** - * Get datasource action name - */ - public static final String NAME = "cluster:admin/security_analytics/datasource/get"; - - private GetDatasourceAction() { - super(NAME, GetDatasourceResponse::new); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceTransportAction.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceTransportAction.java deleted file mode 100644 index cb1419517..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/GetDatasourceTransportAction.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.OpenSearchException; -import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.common.inject.Inject; -import org.opensearch.core.action.ActionListener; -import org.opensearch.index.IndexNotFoundException; -import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource; -import org.opensearch.tasks.Task; -import org.opensearch.transport.TransportService; - -import java.util.Collections; -import java.util.List; - -/** - * Transport action to get datasource - */ -public class GetDatasourceTransportAction extends HandledTransportAction { - private final DatasourceDao datasourceDao; - - /** - * Default constructor - * @param transportService the transport service - * @param actionFilters the action filters - * @param datasourceDao the datasource facade - */ - @Inject - public GetDatasourceTransportAction( - final TransportService transportService, - final ActionFilters actionFilters, - final DatasourceDao datasourceDao - ) { - super(GetDatasourceAction.NAME, transportService, actionFilters, GetDatasourceRequest::new); - this.datasourceDao = datasourceDao; - } - - @Override - protected void doExecute(final Task task, final GetDatasourceRequest request, final ActionListener listener) { - if (shouldGetAllDatasource(request)) { - // We don't expect too many data sources. Therefore, querying all data sources without pagination should be fine. - datasourceDao.getAllDatasources(newActionListener(listener)); - } else { - datasourceDao.getDatasources(request.getNames(), newActionListener(listener)); - } - } - - private boolean shouldGetAllDatasource(final GetDatasourceRequest request) { - if (request.getNames() == null) { - throw new OpenSearchException("names in a request should not be null"); - } - - return request.getNames().length == 0 || (request.getNames().length == 1 && "_all".equals(request.getNames()[0])); - } - - protected ActionListener> newActionListener(final ActionListener listener) { - return new ActionListener<>() { - @Override - public void onResponse(final List datasources) { - listener.onResponse(new GetDatasourceResponse(datasources)); - } - - @Override - public void onFailure(final Exception e) { - if (e instanceof IndexNotFoundException) { - listener.onResponse(new GetDatasourceResponse(Collections.emptyList())); - return; - } - listener.onFailure(e); - } - }; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceRequest.java deleted file mode 100644 index dac67ed43..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/PutDatasourceRequest.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.List; -import java.util.Locale; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.ParseField; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.xcontent.ObjectParser; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceManifest; -import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; - -/** - * Threat intel datasource creation request - */ -public class PutDatasourceRequest extends ActionRequest { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - - public static final ParseField FEED_FORMAT_FIELD = new ParseField("feed_format"); - public static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); - public static final ParseField FEED_NAME_FIELD = new ParseField("feed_name"); - public static final ParseField DESCRIPTION_FIELD = new ParseField("description"); - public static final ParseField ORGANIZATION_FIELD = new ParseField("organization"); - public static final ParseField CONTAINED_IOCS_FIELD = new ParseField("contained_iocs_field"); - public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); - private static final ParameterValidator VALIDATOR = new ParameterValidator(); - - /** - * @param name the datasource name - * @return the datasource name - */ - private String name; - - private String feedFormat; - - /** - * @param endpoint url to a manifest file for a datasource - * @return url to a manifest file for a datasource - */ - private String endpoint; - - private String feedName; - - private String description; - - private String organization; - - private List contained_iocs_field; - - public void setFeedFormat(String feedFormat) { - this.feedFormat = feedFormat; - } - - public void setThisEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - public void setFeedName(String feedName) { - this.feedName = feedName; - } - - public void setDescription(String description) { - this.description = description; - } - - public void setOrganization(String organization) { - this.organization = organization; - } - - public void setContained_iocs_field(List contained_iocs_field) { - this.contained_iocs_field = contained_iocs_field; - } - - public List getContained_iocs_field() { - return contained_iocs_field; - } - - public String getFeedFormat() { - return feedFormat; - } - - public String getFeedName() { - return feedName; - } - - @Override - public String getDescription() { - return description; - } - - public String getOrganization() { - return organization; - } - /** - * @param updateInterval update interval of a datasource - * @return update interval of a datasource - */ - private TimeValue updateInterval; - - /** - * Parser of a datasource - */ - public static final ObjectParser PARSER; - static { - PARSER = new ObjectParser<>("put_datasource"); - PARSER.declareString((request, val) -> request.setFeedFormat(val), FEED_FORMAT_FIELD); - PARSER.declareString((request, val) -> request.setThisEndpoint(val), ENDPOINT_FIELD); - PARSER.declareString((request, val) -> request.setFeedName(val), FEED_NAME_FIELD); - PARSER.declareString((request, val) -> request.setDescription(val), DESCRIPTION_FIELD); - PARSER.declareString((request, val) -> request.setOrganization(val), ORGANIZATION_FIELD); -// PARSER.declareStringArray((request, val[]) -> request.setContained_iocs_field(val), CONTAINED_IOCS_FIELD); - PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); - } - - /** - * Default constructor - * @param name name of a datasource - */ - public PutDatasourceRequest(final String name) { - this.name = name; - } - - /** - * Constructor with stream input - * @param in the stream input - * @throws IOException IOException - */ - public PutDatasourceRequest(final StreamInput in) throws IOException { - super(in); - this.name = in.readString(); - this.feedFormat = in.readString(); - this.endpoint = in.readString(); - this.feedName = in.readString(); - this.description = in.readString(); - this.organization = in.readString(); - this.contained_iocs_field = in.readStringList(); - this.updateInterval = in.readTimeValue(); - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(name); - out.writeString(feedFormat); - out.writeString(endpoint); - out.writeString(feedName); - out.writeString(description); - out.writeString(organization); - out.writeStringCollection(contained_iocs_field); - out.writeTimeValue(updateInterval); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException errors = new ActionRequestValidationException(); - List errorMsgs = VALIDATOR.validateDatasourceName(name); - if (errorMsgs.isEmpty() == false) { - errorMsgs.stream().forEach(msg -> errors.addValidationError(msg)); - } - validateEndpoint(errors); - validateUpdateInterval(errors); - return errors.validationErrors().isEmpty() ? null : errors; - } - - /** - * Conduct following validation on endpoint - * 1. endpoint format complies with RFC-2396 - * 2. validate manifest file from the endpoint - * - * @param errors the errors to add error messages - */ - private void validateEndpoint(final ActionRequestValidationException errors) { - try { - URL url = new URL(endpoint); - url.toURI(); // Validate URL complies with RFC-2396 - validateManifestFile(url, errors); - } catch (MalformedURLException | URISyntaxException e) { - log.info("Invalid URL[{}] is provided", endpoint, e); - errors.addValidationError("Invalid URL format is provided"); - } - } - - /** - * Conduct following validation on url - * 1. can read manifest file from the endpoint - * 2. the url in the manifest file complies with RFC-2396 - * 3. updateInterval is less than validForInDays value in the manifest file - * - * @param url the url to validate - * @param errors the errors to add error messages - */ - private void validateManifestFile(final URL url, final ActionRequestValidationException errors) { - DatasourceManifest manifest; - try { - manifest = DatasourceManifest.Builder.build(url); - } catch (Exception e) { - log.info("Error occurred while reading a file from {}", url, e); - errors.addValidationError(String.format(Locale.ROOT, "Error occurred while reading a file from %s: %s", url, e.getMessage())); - return; - } - - try { - new URL(manifest.getUrl()).toURI(); // Validate URL complies with RFC-2396 - } catch (MalformedURLException | URISyntaxException e) { - log.info("Invalid URL[{}] is provided for url field in the manifest file", manifest.getUrl(), e); - errors.addValidationError("Invalid URL format is provided for url field in the manifest file"); - return; - } - -// if (manifest.getValidForInDays() != null && updateInterval.days() >= manifest.getValidForInDays()) { -// errors.addValidationError( -// String.format( -// Locale.ROOT, -// "updateInterval %d should be smaller than %d", -// updateInterval.days(), -// manifest.getValidForInDays() -// ) -// ); -// } - } - - /** - * Validate updateInterval is equal or larger than 1 - * - * @param errors the errors to add error messages - */ - private void validateUpdateInterval(final ActionRequestValidationException errors) { - if (updateInterval.compareTo(TimeValue.timeValueDays(1)) < 0) { - errors.addValidationError("Update interval should be equal to or larger than 1 day"); - } - } - - public String getName() { - return name; - } - - public String getEndpoint() { - return this.endpoint; - } - - public void setEndpoint(String newEndpoint) { - this.endpoint = newEndpoint; - } - - public TimeValue getUpdateInterval() { - return this.updateInterval; - } - - public void setUpdateInterval(TimeValue timeValue) { - this.updateInterval = timeValue; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestDeleteDatasourceHandler.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestDeleteDatasourceHandler.java deleted file mode 100644 index 3da4c4abc..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestDeleteDatasourceHandler.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.RestToXContentListener; - -import java.io.IOException; -import java.util.List; -import java.util.Locale; - -import static org.opensearch.rest.RestRequest.Method.DELETE; - -/** - * Rest handler for threat intel datasource delete request - */ -public class RestDeleteDatasourceHandler extends BaseRestHandler { - private static final String ACTION_NAME = "threatintel_datasource_delete"; - private static final String PARAMS_NAME = "name"; - - @Override - public String getName() { - return ACTION_NAME; - } - - @Override - protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final String name = request.param(PARAMS_NAME); - final DeleteDatasourceRequest deleteDatasourceRequest = new DeleteDatasourceRequest(name); - - return channel -> client.executeLocally( - DeleteDatasourceAction.INSTANCE, - deleteDatasourceRequest, - new RestToXContentListener<>(channel) - ); - } - - @Override - public List routes() { - String path = String.join("/", "/_plugins/_security_analytics", String.format(Locale.ROOT, "threatintel/datasource/{%s}", PARAMS_NAME)); - return List.of(new Route(DELETE, path)); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestGetDatasourceHandler.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestGetDatasourceHandler.java deleted file mode 100644 index ddbecdad5..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestGetDatasourceHandler.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.core.common.Strings; -import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.RestToXContentListener; - -import java.util.List; - -import static org.opensearch.rest.RestRequest.Method.GET; - -/** - * Rest handler for threat intel datasource get request - */ -public class RestGetDatasourceHandler extends BaseRestHandler { - private static final String ACTION_NAME = "threatintel_datasource_get"; - - @Override - public String getName() { - return ACTION_NAME; - } - - @Override - protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { - final String[] names = request.paramAsStringArray("name", Strings.EMPTY_ARRAY); - final GetDatasourceRequest getDatasourceRequest = new GetDatasourceRequest(names); - - return channel -> client.executeLocally(GetDatasourceAction.INSTANCE, getDatasourceRequest, new RestToXContentListener<>(channel)); - } - - @Override - public List routes() { - return List.of( - new Route(GET, String.join("/", "/_plugins/_security_analytics", "threatintel/datasource")), - new Route(GET, String.join("/", "/_plugins/_security_analytics", "threatintel/datasource/{name}")) - ); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestPutDatasourceHandler.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestPutDatasourceHandler.java deleted file mode 100644 index 5c9ecd7b4..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestPutDatasourceHandler.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.RestToXContentListener; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelSettings; - -import java.io.IOException; -import java.util.List; - -import static org.opensearch.rest.RestRequest.Method.PUT; - -/** - * Rest handler for threat intel datasource creation - * - * This handler handles a request of - * PUT /_plugins/security_analytics/threatintel/datasource/{id} - * { - * "endpoint": {endpoint}, - * "update_interval_in_days": 3 - * } - * - * When request is received, it will create a datasource by downloading threat intel feed from the endpoint. - * After the creation of datasource is completed, it will schedule the next update task after update_interval_in_days. - * - */ -public class RestPutDatasourceHandler extends BaseRestHandler { - private static final String ACTION_NAME = "threatintel_datasource_put"; - private final ClusterSettings clusterSettings; - - public RestPutDatasourceHandler(final ClusterSettings clusterSettings) { - this.clusterSettings = clusterSettings; - } - - @Override - public String getName() { - return ACTION_NAME; - } - - @Override - protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final PutDatasourceRequest putDatasourceRequest = new PutDatasourceRequest(request.param("name")); - if (request.hasContentOrSourceParam()) { - try (XContentParser parser = request.contentOrSourceParamParser()) { - PutDatasourceRequest.PARSER.parse(parser, putDatasourceRequest, null); - } - } - if (putDatasourceRequest.getEndpoint() == null) { - putDatasourceRequest.setEndpoint(clusterSettings.get(ThreatIntelSettings.DATASOURCE_ENDPOINT)); - } - if (putDatasourceRequest.getUpdateInterval() == null) { - putDatasourceRequest.setUpdateInterval(TimeValue.timeValueDays(clusterSettings.get(ThreatIntelSettings.DATASOURCE_UPDATE_INTERVAL))); - } - return channel -> client.executeLocally(PutDatasourceAction.INSTANCE, putDatasourceRequest, new RestToXContentListener<>(channel)); - } - - @Override - public List routes() { - String path = String.join("/", "/_plugins/_security_analytics", "threatintel/datasource/{name}"); - return List.of(new Route(PUT, path)); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestUpdateDatasourceHandler.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestUpdateDatasourceHandler.java deleted file mode 100644 index 3f755670f..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/RestUpdateDatasourceHandler.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.RestToXContentListener; - -import java.io.IOException; -import java.util.List; - -import static org.opensearch.rest.RestRequest.Method.PUT; - -/** - * Rest handler for threat intel datasource update request - */ -public class RestUpdateDatasourceHandler extends BaseRestHandler { - private static final String ACTION_NAME = "threatintel_datasource_update"; - - @Override - public String getName() { - return ACTION_NAME; - } - - @Override - protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final UpdateDatasourceRequest updateDatasourceRequest = new UpdateDatasourceRequest(request.param("name")); - if (request.hasContentOrSourceParam()) { - try (XContentParser parser = request.contentOrSourceParamParser()) { - UpdateDatasourceRequest.PARSER.parse(parser, updateDatasourceRequest, null); - } - } - return channel -> client.executeLocally( - UpdateDatasourceAction.INSTANCE, - updateDatasourceRequest, - new RestToXContentListener<>(channel) - ); - } - - @Override - public List routes() { - String path = String.join("/", "/_plugins/_security_analytics", "threatintel/datasource/{name}/_settings"); - return List.of(new Route(PUT, path)); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceRequest.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceRequest.java deleted file mode 100644 index 7d70f45aa..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceRequest.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.ParseField; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.xcontent.ObjectParser; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceManifest; -import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Locale; - -/** - * threat intel datasource update request - */ -public class UpdateDatasourceRequest extends ActionRequest { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - - public static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); - public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); - private static final int MAX_DATASOURCE_NAME_BYTES = 255; - private static final ParameterValidator VALIDATOR = new ParameterValidator(); - - /** - * @param name the datasource name - * @return the datasource name - */ - private String name; - - /** - * @param endpoint url to a manifest file for a datasource - * @return url to a manifest file for a datasource - */ - private String endpoint; - - /** - * @param updateInterval update interval of a datasource - * @return update interval of a datasource - */ - private TimeValue updateInterval; - - /** - * Parser of a datasource - */ - public static final ObjectParser PARSER; - static { - PARSER = new ObjectParser<>("update_datasource"); - PARSER.declareString((request, val) -> request.setEndpoint(val), ENDPOINT_FIELD); - PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); - } - - public String getName() { - return name; - } - public String getEndpoint() { - return endpoint; - } - private void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - public TimeValue getUpdateInterval() { - return updateInterval; - } - - private void setUpdateInterval(TimeValue updateInterval){ - this.updateInterval = updateInterval; - } - - /** - * Constructor - * @param name name of a datasource - */ - public UpdateDatasourceRequest(final String name) { - this.name = name; - } - - /** - * Constructor - * @param in the stream input - * @throws IOException IOException - */ - public UpdateDatasourceRequest(final StreamInput in) throws IOException { - super(in); - this.name = in.readString(); - this.endpoint = in.readOptionalString(); - this.updateInterval = in.readOptionalTimeValue(); - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(name); - out.writeOptionalString(endpoint); - out.writeOptionalTimeValue(updateInterval); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException errors = new ActionRequestValidationException(); - if (VALIDATOR.validateDatasourceName(name).isEmpty() == false) { - errors.addValidationError("no such datasource exist"); - } - if (endpoint == null && updateInterval == null) { - errors.addValidationError("no values to update"); - } - - validateEndpoint(errors); - validateUpdateInterval(errors); - - return errors.validationErrors().isEmpty() ? null : errors; - } - - /** - * Conduct following validation on endpoint - * 1. endpoint format complies with RFC-2396 - * 2. validate manifest file from the endpoint - * - * @param errors the errors to add error messages - */ - private void validateEndpoint(final ActionRequestValidationException errors) { - if (endpoint == null) { - return; - } - - try { - URL url = new URL(endpoint); - url.toURI(); // Validate URL complies with RFC-2396 - validateManifestFile(url, errors); - } catch (MalformedURLException | URISyntaxException e) { - log.info("Invalid URL[{}] is provided", endpoint, e); - errors.addValidationError("Invalid URL format is provided"); - } - } - - /** - * Conduct following validation on url - * 1. can read manifest file from the endpoint - * 2. the url in the manifest file complies with RFC-2396 - * - * @param url the url to validate - * @param errors the errors to add error messages - */ - private void validateManifestFile(final URL url, final ActionRequestValidationException errors) { - DatasourceManifest manifest; - try { - manifest = DatasourceManifest.Builder.build(url); - } catch (Exception e) { - log.info("Error occurred while reading a file from {}", url, e); - errors.addValidationError(String.format(Locale.ROOT, "Error occurred while reading a file from %s: %s", url, e.getMessage())); - return; - } - - try { - new URL(manifest.getUrl()).toURI(); // Validate URL complies with RFC-2396 - } catch (MalformedURLException | URISyntaxException e) { - log.info("Invalid URL[{}] is provided for url field in the manifest file", manifest.getUrl(), e); - errors.addValidationError("Invalid URL format is provided for url field in the manifest file"); - } - } - - /** - * Validate updateInterval is equal or larger than 1 - * - * @param errors the errors to add error messages - */ - private void validateUpdateInterval(final ActionRequestValidationException errors) { - if (updateInterval == null) { - return; - } - - if (updateInterval.compareTo(TimeValue.timeValueDays(1)) < 0) { - errors.addValidationError("Update interval should be equal to or larger than 1 day"); - } - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceTransportAction.java deleted file mode 100644 index 11d99e41c..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/action/UpdateDatasourceTransportAction.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.OpenSearchStatusException; -import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.inject.Inject; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceState; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelLockService; -import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.DatasourceTask; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.DatasourceUpdateService; -import org.opensearch.tasks.Task; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.TransportService; - -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.Locale; - -/** - * Transport action to update datasource - */ -public class UpdateDatasourceTransportAction extends HandledTransportAction { - private static final long LOCK_DURATION_IN_SECONDS = 300l; - private final ThreatIntelLockService lockService; - private final DatasourceDao datasourceDao; - private final DatasourceUpdateService datasourceUpdateService; - private final ThreadPool threadPool; - - /** - * Constructor - * - * @param transportService the transport service - * @param actionFilters the action filters - * @param lockService the lock service - * @param datasourceDao the datasource facade - * @param datasourceUpdateService the datasource update service - */ - @Inject - public UpdateDatasourceTransportAction( - final TransportService transportService, - final ActionFilters actionFilters, - final ThreatIntelLockService lockService, - final DatasourceDao datasourceDao, - final DatasourceUpdateService datasourceUpdateService, - final ThreadPool threadPool - ) { - super(UpdateDatasourceAction.NAME, transportService, actionFilters, UpdateDatasourceRequest::new); - this.lockService = lockService; - this.datasourceUpdateService = datasourceUpdateService; - this.datasourceDao = datasourceDao; - this.threadPool = threadPool; - } - - /** - * Get a lock and update datasource - * - * @param task the task - * @param request the request - * @param listener the listener - */ - @Override - protected void doExecute(final Task task, final UpdateDatasourceRequest request, final ActionListener listener) { - lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { - if (lock == null) { - listener.onFailure( - new OpenSearchStatusException("Another processor is holding a lock on the resource. Try again later", RestStatus.BAD_REQUEST) - ); - return; - } - try { - // TODO: makes every sub-methods as async call to avoid using a thread in generic pool - threadPool.generic().submit(() -> { - try { - Datasource datasource = datasourceDao.getDatasource(request.getName()); - if (datasource == null) { - throw new ResourceNotFoundException("no such datasource exist"); - } - if (DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "data source is not in an [%s] state", DatasourceState.AVAILABLE) - ); - } - validate(request, datasource); - updateIfChanged(request, datasource); - lockService.releaseLock(lock); - listener.onResponse(new AcknowledgedResponse(true)); - } catch (Exception e) { - lockService.releaseLock(lock); - listener.onFailure(e); - } - }); - } catch (Exception e) { - lockService.releaseLock(lock); - listener.onFailure(e); - } - }, exception -> listener.onFailure(exception))); - } - - private void updateIfChanged(final UpdateDatasourceRequest request, final Datasource datasource) { - boolean isChanged = false; - if (isEndpointChanged(request, datasource)) { - datasource.setEndpoint(request.getEndpoint()); - isChanged = true; - } - if (isUpdateIntervalChanged(request)) { - datasource.setSchedule(new IntervalSchedule(Instant.now(), (int) request.getUpdateInterval().getDays(), ChronoUnit.DAYS)); - datasource.setTask(DatasourceTask.ALL); - isChanged = true; - } - - if (isChanged) { - datasourceDao.updateDatasource(datasource); - } - } - - /** - * Additional validation based on an existing datasource - * - * Basic validation is done in UpdateDatasourceRequest#validate - * In this method we do additional validation based on an existing datasource - * - * 1. Check the compatibility of new fields and old fields - * 2. Check the updateInterval is less than validForInDays in datasource - * - * This method throws exception if one of validation fails. - * - * @param request the update request - * @param datasource the existing datasource - * @throws IOException the exception - */ - private void validate(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { - validateFieldsCompatibility(request, datasource); - } - - private void validateFieldsCompatibility(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { - if (isEndpointChanged(request, datasource) == false) { - return; - } - - List fields = datasourceUpdateService.getHeaderFields(request.getEndpoint()); - if (datasource.isCompatible(fields) == false) { -// throw new IncompatibleDatasourceException( -// "new fields [{}] does not contain all old fields [{}]", -// fields.toString(), -// datasource.getDatabase().getFields().toString() -// ); - throw new OpenSearchStatusException("new fields does not contain all old fields", RestStatus.BAD_REQUEST); - } - } - - private boolean isEndpointChanged(final UpdateDatasourceRequest request, final Datasource datasource) { - return request.getEndpoint() != null && request.getEndpoint().equals(datasource.getEndpoint()) == false; - } - - /** - * Update interval is changed as long as user provide one because - * start time will get updated even if the update interval is same as current one. - * - * @param request the update datasource request - * @return true if update interval is changed, and false otherwise - */ - private boolean isUpdateIntervalChanged(final UpdateDatasourceRequest request) { - return request.getUpdateInterval() != null; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceManifest.java b/src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceManifest.java deleted file mode 100644 index 1417c8a36..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceManifest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.securityanalytics.threatIntel.common; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URL; -import java.net.URLConnection; -import java.nio.CharBuffer; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Locale; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.SpecialPermission; -import org.opensearch.Version; -import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.ParseField; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.ConstructingObjectParser; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; - -/** - * Threat intel datasource manifest file object - * - * Manifest file is stored in an external endpoint. OpenSearch read the file and store values it in this object. - */ -public class DatasourceManifest { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - - private static final ParseField URL_FIELD = new ParseField("url"); //url for csv threat intel feed - private static final ParseField DB_NAME_FIELD = new ParseField("db_name"); // name of the db (csv file for now) - private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); //not using for now - private static final ParseField ORGANIZATION_FIELD = new ParseField("organization"); //not using for now - private static final ParseField DESCRIPTION_FIELD = new ParseField("description"); //not using for now - private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at_in_epoch_milli"); //not using for now - - /** - * @param url URL of a ZIP file containing a database - * @return URL of a ZIP file containing a database - */ - private String url; - - /** - * @param dbName A database file name inside the ZIP file - * @return A database file name inside the ZIP file - */ - private String dbName; - /** - * @param sha256Hash SHA256 hash value of a database file - * @return SHA256 hash value of a database file - */ - private String sha256Hash; - - /** - * @param organization A database organization name - * @return A database organization name - */ - private String organization; - /** - * @param description A description of the database - * @return A description of a database - */ - private String description; - /** - * @param updatedAt A date when the database was updated - * @return A date when the database was updated - */ - private Long updatedAt; - - public String getUrl() { - return this.url; - } - public String getDbName() { - return dbName; - } - - public String getOrganization() { - return organization; - } - - public String getSha256Hash() { - return sha256Hash; - } - - public String getDescription() { - return description; - } - - public Long getUpdatedAt() { - return updatedAt; - } - - public DatasourceManifest(final String url, final String dbName) { - this.url = url; - this.dbName = dbName; - } - - /** - * Datasource manifest parser - */ - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "datasource_manifest", - true, - args -> { - String url = (String) args[0]; - String dbName = (String) args[1]; - return new DatasourceManifest(url, dbName); - } - ); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), URL_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), DB_NAME_FIELD); - } - - /** - * Datasource manifest builder - */ - public static class Builder { - private static final int MANIFEST_FILE_MAX_BYTES = 1024 * 8; - - /** - * Build DatasourceManifest from a given url - * - * @param url url to downloads a manifest file - * @return DatasourceManifest representing the manifest file - */ - @SuppressForbidden(reason = "Need to connect to http endpoint to read manifest file") // change permissions - public static DatasourceManifest build(final URL url) { - SpecialPermission.check(); - return AccessController.doPrivileged((PrivilegedAction) () -> { - try { - URLConnection connection = url.openConnection(); - return internalBuild(connection); - } catch (IOException e) { - log.error("Runtime exception connecting to the manifest file", e); - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO - } - }); - } - - @SuppressForbidden(reason = "Need to connect to http endpoint to read manifest file") - protected static DatasourceManifest internalBuild(final URLConnection connection) throws IOException { - connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); - InputStreamReader inputStreamReader = new InputStreamReader(connection.getInputStream()); - try (BufferedReader reader = new BufferedReader(inputStreamReader)) { - CharBuffer charBuffer = CharBuffer.allocate(MANIFEST_FILE_MAX_BYTES); - reader.read(charBuffer); - charBuffer.flip(); - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.IGNORE_DEPRECATIONS, - charBuffer.toString() - ); - return PARSER.parse(parser, null); - } - } - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceState.java b/src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceState.java deleted file mode 100644 index a516b1d34..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/common/DatasourceState.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.common; - -/** - * Threat intel datasource state - * - * When data source is created, it starts with CREATING state. Once the first threat intel feed is generated, the state changes to AVAILABLE. - * Only when the first threat intel feed generation failed, the state changes to CREATE_FAILED. - * Subsequent threat intel feed failure won't change data source state from AVAILABLE to CREATE_FAILED. - * When delete request is received, the data source state changes to DELETING. - * - * State changed from left to right for the entire lifecycle of a datasource - * (CREATING) to (CREATE_FAILED or AVAILABLE) to (DELETING) - * - */ -public enum DatasourceState { - /** - * Data source is being created - */ - CREATING, - /** - * Data source is ready to be used - */ - AVAILABLE, - /** - * Data source creation failed - */ - CREATE_FAILED, - /** - * Data source is being deleted - */ - DELETING -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ParameterValidator.java b/src/main/java/org/opensearch/securityanalytics/threatintel/common/ParameterValidator.java index 13276975c..25e40837c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ParameterValidator.java +++ b/src/main/java/org/opensearch/securityanalytics/threatintel/common/ParameterValidator.java @@ -25,7 +25,7 @@ public class ParameterValidator { * @param datasourceName datasource name * @return Error messages. Empty list if there is no violation. */ - public List validateDatasourceName(final String datasourceName) { + public List validateTIFJobName(final String datasourceName) { List errorMsgs = new ArrayList<>(); if (StringUtils.isBlank(datasourceName)) { errorMsgs.add("datasource name must not be empty"); diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelSettings.java b/src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelSettings.java deleted file mode 100644 index 1d649e0b6..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ThreatIntelSettings.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.common; - -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.securityanalytics.model.DetectorTrigger; - -/** - * Settings for threat intel datasource operations - */ -public class ThreatIntelSettings { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - - - /** - * Default endpoint to be used in threat intel feed datasource creation API - */ - public static final Setting DATASOURCE_ENDPOINT = Setting.simpleString( - "plugins.security_analytics.threatintel.datasource.endpoint", - "https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json", //TODO fix this endpoint - new DatasourceEndpointValidator(), - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Default update interval to be used in threat intel datasource creation API - */ - public static final Setting DATASOURCE_UPDATE_INTERVAL = Setting.longSetting( - "plugins.security_analytics.threatintel.datasource.update_interval_in_days", - 3l, - 1l, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Bulk size for indexing threat intel feed data - */ - public static final Setting BATCH_SIZE = Setting.intSetting( - "plugins.security_analytics.threatintel.datasource.batch_size", - 10000, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Timeout value for threat intel processor - */ - public static final Setting THREAT_INTEL_TIMEOUT = Setting.timeSetting( - "plugins.security_analytics.threat_intel_timeout", - TimeValue.timeValueSeconds(30), - TimeValue.timeValueSeconds(1), - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Max size for threat intel feed cache - */ - public static final Setting CACHE_SIZE = Setting.longSetting( - "plugins.security_analytics.threatintel.processor.cache_size", - 1000, - 0, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Return all settings of threat intel feature - * @return a list of all settings for threat intel feature - */ - public static final List> settings() { - return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, THREAT_INTEL_TIMEOUT); - } - - /** - * Visible for testing - */ - protected static class DatasourceEndpointValidator implements Setting.Validator { - @Override - public void validate(final String value) { - try { - new URL(value).toURI(); - } catch (MalformedURLException | URISyntaxException e) { - log.error("Invalid URL format is provided", e); - throw new IllegalArgumentException("Invalid URL format is provided"); - } - } - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceRunner.java deleted file mode 100644 index 8de306d33..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceRunner.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.jobscheduler; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.securityanalytics.model.DetectorTrigger; - -import java.io.IOException; -import java.time.temporal.ChronoUnit; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; -import java.time.Instant; - -import org.opensearch.securityanalytics.threatIntel.common.DatasourceState; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelExecutor; -import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelLockService; -import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao; -/** - * Datasource update task - * - * This is a background task which is responsible for updating threat intel feed data - */ -public class DatasourceRunner implements ScheduledJobRunner { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - private static DatasourceRunner INSTANCE; - - public static DatasourceRunner getJobRunnerInstance() { - if (INSTANCE != null) { - return INSTANCE; - } - synchronized (DatasourceRunner.class) { - if (INSTANCE != null) { - return INSTANCE; - } - INSTANCE = new DatasourceRunner(); - return INSTANCE; - } - } - - private ClusterService clusterService; - - // threat intel specific variables - private DatasourceUpdateService datasourceUpdateService; - private DatasourceDao datasourceDao; - private ThreatIntelExecutor threatIntelExecutor; - private ThreatIntelLockService lockService; - private boolean initialized; - - private DatasourceRunner() { - // Singleton class, use getJobRunner method instead of constructor - } - - public void initialize( - final ClusterService clusterService, - final DatasourceUpdateService datasourceUpdateService, - final DatasourceDao datasourceDao, - final ThreatIntelExecutor threatIntelExecutor, - final ThreatIntelLockService threatIntelLockService - ) { - this.clusterService = clusterService; - this.datasourceUpdateService = datasourceUpdateService; - this.datasourceDao = datasourceDao; - this.threatIntelExecutor = threatIntelExecutor; - this.lockService = threatIntelLockService; - this.initialized = true; - } - - @Override - public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { - if (initialized == false) { - throw new AssertionError("this instance is not initialized"); - } - - log.info("Update job started for a datasource[{}]", jobParameter.getName()); - if (jobParameter instanceof Datasource == false) { - log.error("Illegal state exception: job parameter is not instance of Datasource"); - throw new IllegalStateException( - "job parameter is not instance of Datasource, type: " + jobParameter.getClass().getCanonicalName() - ); - } - threatIntelExecutor.forDatasourceUpdate().submit(updateDatasourceRunner(jobParameter)); - } - - /** - * Update threat intel feed data - * - * Lock is used so that only one of nodes run this task. - * - * @param jobParameter job parameter - */ - protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParameter) { - return () -> { - Optional lockModel = lockService.acquireLock( - jobParameter.getName(), - ThreatIntelLockService.LOCK_DURATION_IN_SECONDS - ); - if (lockModel.isEmpty()) { - log.error("Failed to update. Another processor is holding a lock for datasource[{}]", jobParameter.getName()); - return; - } - - LockModel lock = lockModel.get(); - try { - updateDatasource(jobParameter, lockService.getRenewLockRunnable(new AtomicReference<>(lock))); - } catch (Exception e) { - log.error("Failed to update datasource[{}]", jobParameter.getName(), e); - } finally { - lockService.releaseLock(lock); - } - }; - } - - protected void updateDatasource(final ScheduledJobParameter jobParameter, final Runnable renewLock) throws IOException { - Datasource datasource = datasourceDao.getDatasource(jobParameter.getName()); - /** - * If delete request comes while update task is waiting on a queue for other update tasks to complete, - * because update task for this datasource didn't acquire a lock yet, delete request is processed. - * When it is this datasource's turn to run, it will find that the datasource is deleted already. - * Therefore, we stop the update process when data source does not exist. - */ - if (datasource == null) { - log.info("Datasource[{}] does not exist", jobParameter.getName()); - return; - } - - if (DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { - log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.AVAILABLE, datasource.getState()); - datasource.disable(); - datasource.getUpdateStats().setLastFailedAt(Instant.now()); - datasourceDao.updateDatasource(datasource); - return; - } - try { - datasourceUpdateService.deleteUnusedIndices(datasource); - if (DatasourceTask.DELETE_UNUSED_INDICES.equals(datasource.getTask()) == false) { - datasourceUpdateService.updateOrCreateThreatIntelFeedData(datasource, renewLock); - } - datasourceUpdateService.deleteUnusedIndices(datasource); - } catch (Exception e) { - log.error("Failed to update datasource for {}", datasource.getName(), e); - datasource.getUpdateStats().setLastFailedAt(Instant.now()); - datasourceDao.updateDatasource(datasource); - } finally { //post processing - datasourceUpdateService.updateDatasource(datasource, datasource.getSchedule(), DatasourceTask.ALL); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceUpdateService.java deleted file mode 100644 index 5a24c5a84..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/jobscheduler/DatasourceUpdateService.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.jobscheduler; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.net.URL; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; - -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.opensearch.OpenSearchException; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.ClusterSettings; - -import org.opensearch.core.rest.RestStatus; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceManifest; -import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.common.DatasourceState; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; - -public class DatasourceUpdateService { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - - private static final int SLEEP_TIME_IN_MILLIS = 5000; // 5 seconds - private static final int MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS = 10 * 60 * 60 * 1000; // 10 hours - private final ClusterService clusterService; - private final ClusterSettings clusterSettings; - private final DatasourceDao datasourceDao; - private final ThreatIntelFeedDataService threatIntelFeedDataService; - - public DatasourceUpdateService( - final ClusterService clusterService, - final DatasourceDao datasourceDao, - final ThreatIntelFeedDataService threatIntelFeedDataService - ) { - this.clusterService = clusterService; - this.clusterSettings = clusterService.getClusterSettings(); - this.datasourceDao = datasourceDao; - this.threatIntelFeedDataService = threatIntelFeedDataService; - } - - /** - * Update threat intel feed data - * - * The first column is ip range field regardless its header name. - * Therefore, we don't store the first column's header name. - * - * @param datasource the datasource - * @param renewLock runnable to renew lock - * - * @throws IOException - */ - public void updateOrCreateThreatIntelFeedData(final Datasource datasource, final Runnable renewLock) throws IOException { - URL url = new URL(datasource.getEndpoint()); - DatasourceManifest manifest = DatasourceManifest.Builder.build(url); - - if (shouldUpdate(datasource, manifest) == false) { - log.info("Skipping threat intel feed database update. Update is not required for {}", datasource.getName()); - datasource.getUpdateStats().setLastSkippedAt(Instant.now()); - datasourceDao.updateDatasource(datasource); - return; - } - - Instant startTime = Instant.now(); - String indexName = setupIndex(datasource); - String[] header; - List fieldsToStore; - try (CSVParser reader = threatIntelFeedDataService.getDatabaseReader(manifest)) { - CSVRecord headerLine = reader.iterator().next(); - header = validateHeader(headerLine).values(); - fieldsToStore = Arrays.asList(header).subList(1, header.length); - if (datasource.isCompatible(fieldsToStore) == false) { - log.error("Exception: new fields does not contain all old fields"); - throw new OpenSearchException( - "new fields [{}] does not contain all old fields [{}]", - fieldsToStore.toString(), - datasource.getDatabase().getFields().toString() - ); - } - threatIntelFeedDataService.saveThreatIntelFeedData(indexName, header, reader.iterator(), renewLock); - } - - waitUntilAllShardsStarted(indexName, MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS); - Instant endTime = Instant.now(); - updateDatasourceAsSucceeded(indexName, datasource, manifest, fieldsToStore, startTime, endTime); // then I update the datasource - } - - - /** - * We wait until all shards are ready to serve search requests before updating datasource metadata to - * point to a new index so that there won't be latency degradation during threat intel feed data update - * - * @param indexName the indexName - */ - protected void waitUntilAllShardsStarted(final String indexName, final int timeout) { - Instant start = Instant.now(); - try { - while (Instant.now().toEpochMilli() - start.toEpochMilli() < timeout) { - if (clusterService.state().routingTable().allShards(indexName).stream().allMatch(shard -> shard.started())) { - return; - } - Thread.sleep(SLEEP_TIME_IN_MILLIS); - } - throw new OpenSearchException( - "index[{}] replication did not complete after {} millis", - MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS - ); - } catch (InterruptedException e) { - log.error("runtime exception", e); - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO - } - } - - /** - * Return header fields of threat intel feed data with given url of a manifest file - * - * The first column is ip range field regardless its header name. - * Therefore, we don't store the first column's header name. - * - * @param manifestUrl the url of a manifest file - * @return header fields of threat intel feed - */ - public List getHeaderFields(String manifestUrl) throws IOException { - URL url = new URL(manifestUrl); - DatasourceManifest manifest = DatasourceManifest.Builder.build(url); - - try (CSVParser reader = threatIntelFeedDataService.getDatabaseReader(manifest)) { - String[] fields = reader.iterator().next().values(); - return Arrays.asList(fields).subList(1, fields.length); - } - } - - /** - * Delete all indices except the one which are being used - * - * @param datasource - */ - public void deleteUnusedIndices(final Datasource datasource) { - try { - List indicesToDelete = datasource.getIndices() - .stream() - .filter(index -> index.equals(datasource.currentIndexName()) == false) - .collect(Collectors.toList()); - - List deletedIndices = deleteIndices(indicesToDelete); - - if (deletedIndices.isEmpty() == false) { - datasource.getIndices().removeAll(deletedIndices); - datasourceDao.updateDatasource(datasource); - } - } catch (Exception e) { - log.error("Failed to delete old indices for {}", datasource.getName(), e); - } - } - - /** - * Update datasource with given systemSchedule and task - * - * @param datasource datasource to update - * @param systemSchedule new system schedule value - * @param task new task value - */ - public void updateDatasource(final Datasource datasource, final IntervalSchedule systemSchedule, final DatasourceTask task) { - boolean updated = false; - if (datasource.getSchedule().equals(systemSchedule) == false) { - datasource.setSchedule(systemSchedule); - updated = true; - } - - if (datasource.getTask().equals(task) == false) { - datasource.setTask(task); - updated = true; - } - - if (updated) { - datasourceDao.updateDatasource(datasource); - } - } - - private List deleteIndices(final List indicesToDelete) { - List deletedIndices = new ArrayList<>(indicesToDelete.size()); - for (String index : indicesToDelete) { - if (clusterService.state().metadata().hasIndex(index) == false) { - deletedIndices.add(index); - continue; - } - - try { - threatIntelFeedDataService.deleteThreatIntelDataIndex(index); - deletedIndices.add(index); - } catch (Exception e) { - log.error("Failed to delete an index [{}]", index, e); - } - } - return deletedIndices; - } - - /** - * Validate header - * - * 1. header should not be null - * 2. the number of values in header should be more than one - * - * @param header the header - * @return CSVRecord the input header - */ - private CSVRecord validateHeader(CSVRecord header) { - if (header == null) { - throw new OpenSearchException("threat intel feed database is empty"); - } - if (header.values().length < 2) { - throw new OpenSearchException("threat intel feed database should have at least two fields"); - } - return header; - } - - /*** - * Update datasource as succeeded - * - * @param manifest the manifest - * @param datasource the datasource - */ - private void updateDatasourceAsSucceeded( - final String newIndexName, - final Datasource datasource, - final DatasourceManifest manifest, - final List fields, - final Instant startTime, - final Instant endTime - ) { - datasource.setCurrentIndex(newIndexName); - datasource.setDatabase(manifest, fields); - datasource.getUpdateStats().setLastSucceededAt(endTime); - datasource.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); - datasource.enable(); - datasource.setState(DatasourceState.AVAILABLE); - datasourceDao.updateDatasource(datasource); - log.info( - "threat intel feed database creation succeeded for {} and took {} seconds", - datasource.getName(), - Duration.between(startTime, endTime) - ); - } - - /*** - * Setup index to add a new threat intel feed data - * - * @param datasource the datasource - * @return new index name - */ - private String setupIndex(final Datasource datasource) { - String indexName = datasource.newIndexName(UUID.randomUUID().toString()); - datasource.getIndices().add(indexName); - datasourceDao.updateDatasource(datasource); - threatIntelFeedDataService.createIndexIfNotExists(indexName); - return indexName; - } - - /** - * Determine if update is needed or not - * - * Update is needed when all following conditions are met - * 1. updatedAt value in datasource is equal or before updateAt value in manifest - * 2. SHA256 hash value in datasource is different with SHA256 hash value in manifest - * - * @param datasource - * @param manifest - * @return - */ - private boolean shouldUpdate(final Datasource datasource, final DatasourceManifest manifest) { - if (datasource.getDatabase().getUpdatedAt() != null - && datasource.getDatabase().getUpdatedAt().toEpochMilli() > manifest.getUpdatedAt()) { - return false; - } - -// if (manifest.getSha256Hash().equals(datasource.getDatabase().getSha256Hash())) { -// return false; -// } - return true; - } -} diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json new file mode 100644 index 000000000..5e039928d --- /dev/null +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -0,0 +1,118 @@ +{ + "properties": { + "database": { + "properties": { + "feed_id": { + "type": "text" + }, + "feed_name": { + "type": "text" + }, + "feed_format": { + "type": "text" + }, + "endpoint": { + "type": "text" + }, + "description": { + "type": "text" + }, + "organization": { + "type": "text" + }, + "contained_iocs_field": { + "type": "text" + }, + "ioc_col": { + "type": "text" + }, + "fields": { + "type": "text" + } + } + }, + "enabled_time": { + "type": "long" + }, + "indices": { + "type": "text" + }, + "last_update_time": { + "type": "long" + }, + "name": { + "type": "text" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "unit": { + "type": "text" + } + } + } + } + }, + "state": { + "type": "text" + }, + "task": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "update_enabled": { + "type": "boolean" + }, + "update_stats": { + "properties": { + "last_failed_at_in_epoch_millis": { + "type": "long" + }, + "last_processing_time_in_millis": { + "type": "long" + }, + "last_skipped_at_in_epoch_millis": { + "type": "long" + }, + "last_succeeded_at_in_epoch_millis": { + "type": "long" + } + } + }, + "user_schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "unit": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/resources/threatIntelFeedInfo/feodo.yml b/src/main/resources/threatIntelFeedInfo/feodo.yml new file mode 100644 index 000000000..4acbf40e4 --- /dev/null +++ b/src/main/resources/threatIntelFeedInfo/feodo.yml @@ -0,0 +1,6 @@ +url: "https://feodotracker.abuse.ch/downloads/ipblocklist_aggressive.csv" +name: "ipblocklist_aggressive.csv" +feedFormat: "csv" +org: "Feodo" +iocTypes: ["ip"] +description: "" \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java new file mode 100644 index 000000000..c637b448a --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java @@ -0,0 +1,287 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import org.junit.After; +import org.junit.Before; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionType; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Randomness; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.ingest.IngestMetadata; +import org.opensearch.ingest.IngestService; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.utils.LockService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFExecutor; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobTask; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskListener; +import org.opensearch.test.client.NoOpNodeClient; +import org.opensearch.test.rest.RestActionTestCase; +import org.opensearch.threadpool.ThreadPool; + +public abstract class ThreatIntelTestCase extends RestActionTestCase { + @Mock + protected ClusterService clusterService; + @Mock + protected TIFJobUpdateService tifJobUpdateService; + @Mock + protected TIFJobParameterService tifJobParameterService; + @Mock + protected TIFExecutor threatIntelExecutor; + @Mock + protected ThreatIntelFeedDataService threatIntelFeedDataService; + @Mock + protected ClusterState clusterState; + @Mock + protected Metadata metadata; + @Mock + protected IngestService ingestService; + @Mock + protected ActionFilters actionFilters; + @Mock + protected ThreadPool threadPool; + @Mock + protected TIFLockService threatIntelLockService; + @Mock + protected RoutingTable routingTable; + protected IngestMetadata ingestMetadata; + protected NoOpNodeClient client; + protected VerifyingClient verifyingClient; + protected LockService lockService; + protected ClusterSettings clusterSettings; + protected Settings settings; + private AutoCloseable openMocks; + + @Before + public void prepareThreatIntelTestCase() { + openMocks = MockitoAnnotations.openMocks(this); + settings = Settings.EMPTY; + client = new NoOpNodeClient(this.getTestName()); + verifyingClient = spy(new VerifyingClient(this.getTestName())); + clusterSettings = new ClusterSettings(settings, new HashSet<>(SecurityAnalyticsSettings.settings())); + lockService = new LockService(client, clusterService); + ingestMetadata = new IngestMetadata(Collections.emptyMap()); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.state()).thenReturn(clusterState); + when(clusterState.metadata()).thenReturn(metadata); + when(clusterState.getMetadata()).thenReturn(metadata); + when(clusterState.routingTable()).thenReturn(routingTable); + when(ingestService.getClusterService()).thenReturn(clusterService); + when(threadPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); + } + + @After + public void clean() throws Exception { + openMocks.close(); + client.close(); + verifyingClient.close(); + } + + protected TIFJobState randomStateExcept(TIFJobState state) { + assertNotNull(state); + return Arrays.stream(TIFJobState.values()) + .sequential() + .filter(s -> !s.equals(state)) + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(TIFJobState.values().length - 2)); + } + + protected TIFJobState randomState() { + return Arrays.stream(TIFJobState.values()) + .sequential() + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(TIFJobState.values().length - 1)); + } + + protected TIFJobTask randomTask() { + return Arrays.stream(TIFJobTask.values()) + .sequential() + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(TIFJobTask.values().length - 1)); + } + + protected String randomIpAddress() { + return String.format( + Locale.ROOT, + "%d.%d.%d.%d", + Randomness.get().nextInt(255), + Randomness.get().nextInt(255), + Randomness.get().nextInt(255), + Randomness.get().nextInt(255) + ); + } + + protected long randomPositiveLong() { + long value = Randomness.get().nextLong(); + return value < 0 ? -value : value; + } + + /** + * Update interval should be > 0 and < validForInDays. + * For an update test to work, there should be at least one eligible value other than current update interval. + * Therefore, the smallest value for validForInDays is 2. + * Update interval is random value from 1 to validForInDays - 2. + * The new update value will be validForInDays - 1. + */ + protected TIFJobParameter randomTifJobParameter(final Instant updateStartTime) { + Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); + TIFJobParameter tifJobParameter = new TIFJobParameter(); + tifJobParameter.setName(ThreatIntelTestHelper.randomLowerCaseString()); + tifJobParameter.setSchedule( + new IntervalSchedule( + updateStartTime.truncatedTo(ChronoUnit.MILLIS), + 1, + ChronoUnit.DAYS + ) + ); + tifJobParameter.setTask(randomTask()); + tifJobParameter.setState(randomState()); + tifJobParameter.setCurrentIndex(tifJobParameter.newIndexName(UUID.randomUUID().toString())); + tifJobParameter.setIndices(Arrays.asList(ThreatIntelTestHelper.randomLowerCaseString(), ThreatIntelTestHelper.randomLowerCaseString())); + tifJobParameter.getUpdateStats().setLastSkippedAt(now); + tifJobParameter.getUpdateStats().setLastSucceededAt(now); + tifJobParameter.getUpdateStats().setLastFailedAt(now); + tifJobParameter.getUpdateStats().setLastProcessingTimeInMillis(randomPositiveLong()); + tifJobParameter.setLastUpdateTime(now); + if (Randomness.get().nextInt() % 2 == 0) { + tifJobParameter.enable(); + } else { + tifJobParameter.disable(); + } + return tifJobParameter; + } + + protected TIFJobParameter randomTifJobParameter() { + return randomTifJobParameter(Instant.now()); + } + + protected LockModel randomLockModel() { + LockModel lockModel = new LockModel( + ThreatIntelTestHelper.randomLowerCaseString(), + ThreatIntelTestHelper.randomLowerCaseString(), + Instant.now(), + randomPositiveLong(), + false + ); + return lockModel; + } + + /** + * Temporary class of VerifyingClient until this PR(https://github.com/opensearch-project/OpenSearch/pull/7167) + * is merged in OpenSearch core + */ + public static class VerifyingClient extends NoOpNodeClient { + AtomicReference executeVerifier = new AtomicReference<>(); + AtomicReference executeLocallyVerifier = new AtomicReference<>(); + + public VerifyingClient(String testName) { + super(testName); + reset(); + } + + /** + * Clears any previously set verifier functions set by {@link #setExecuteVerifier(BiFunction)} and/or + * {@link #setExecuteLocallyVerifier(BiFunction)}. These functions are replaced with functions which will throw an + * {@link AssertionError} if called. + */ + public void reset() { + executeVerifier.set((arg1, arg2) -> { throw new AssertionError(); }); + executeLocallyVerifier.set((arg1, arg2) -> { throw new AssertionError(); }); + } + + /** + * Sets the function that will be called when {@link #doExecute(ActionType, ActionRequest, ActionListener)} is called. The given + * function should return either a subclass of {@link ActionResponse} or {@code null}. + * @param verifier A function which is called in place of {@link #doExecute(ActionType, ActionRequest, ActionListener)} + */ + public void setExecuteVerifier( + BiFunction, Request, Response> verifier + ) { + executeVerifier.set(verifier); + } + + @Override + public void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) executeVerifier.get().apply(action, request)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Sets the function that will be called when {@link #executeLocally(ActionType, ActionRequest, TaskListener)}is called. The given + * function should return either a subclass of {@link ActionResponse} or {@code null}. + * @param verifier A function which is called in place of {@link #executeLocally(ActionType, ActionRequest, TaskListener)} + */ + public void setExecuteLocallyVerifier( + BiFunction, Request, Response> verifier + ) { + executeLocallyVerifier.set(verifier); + } + + @Override + public Task executeLocally( + ActionType action, + Request request, + ActionListener listener + ) { + listener.onResponse((Response) executeLocallyVerifier.get().apply(action, request)); + return null; + } + + @Override + public Task executeLocally( + ActionType action, + Request request, + TaskListener listener + ) { + listener.onResponse(null, (Response) executeLocallyVerifier.get().apply(action, request)); + return null; + } + + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestHelper.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestHelper.java new file mode 100644 index 000000000..73522053f --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestHelper.java @@ -0,0 +1,120 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.securityanalytics.threatIntel; + +import static org.apache.lucene.tests.util.LuceneTestCase.random; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.opensearch.test.OpenSearchTestCase.randomBoolean; +import static org.opensearch.test.OpenSearchTestCase.randomIntBetween; +import static org.opensearch.test.OpenSearchTestCase.randomNonNegativeLong; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.stream.IntStream; + + +import org.opensearch.OpenSearchException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.replication.ReplicationResponse; +import org.opensearch.common.Randomness; +import org.opensearch.common.UUIDs; +import org.opensearch.common.collect.Tuple; +import org.opensearch.core.index.shard.ShardId; + +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.RandomObjects; + +public class ThreatIntelTestHelper { + + public static final int MAX_SEQ_NO = 10000; + public static final int MAX_PRIMARY_TERM = 10000; + public static final int MAX_VERSION = 10000; + public static final int MAX_SHARD_ID = 100; + + public static final int RANDOM_STRING_MIN_LENGTH = 2; + public static final int RANDOM_STRING_MAX_LENGTH = 16; + + private static String randomString() { + return OpenSearchTestCase.randomAlphaOfLengthBetween(RANDOM_STRING_MIN_LENGTH, RANDOM_STRING_MAX_LENGTH); + } + + public static String randomLowerCaseString() { + return randomString().toLowerCase(Locale.ROOT); + } + + public static List randomLowerCaseStringList() { + List stringList = new ArrayList<>(); + stringList.add(randomLowerCaseString()); + return stringList; + } + + /** + * Returns random {@link IndexResponse} by generating inputs using random functions. + * It is not guaranteed to generate every possible values, and it is not required since + * it is used by the unit test and will not be validated by the cluster. + */ + private static IndexResponse randomIndexResponse() { + String index = randomLowerCaseString(); + String indexUUid = UUIDs.randomBase64UUID(); + int shardId = randomIntBetween(0, MAX_SHARD_ID); + String id = UUIDs.randomBase64UUID(); + long seqNo = randomIntBetween(0, MAX_SEQ_NO); + long primaryTerm = randomIntBetween(0, MAX_PRIMARY_TERM); + long version = randomIntBetween(0, MAX_VERSION); + boolean created = randomBoolean(); + boolean forcedRefresh = randomBoolean(); + Tuple shardInfo = RandomObjects.randomShardInfo(random()); + IndexResponse actual = new IndexResponse(new ShardId(index, indexUUid, shardId), id, seqNo, primaryTerm, version, created); + actual.setForcedRefresh(forcedRefresh); + actual.setShardInfo(shardInfo.v1()); + + return actual; + } + + // Generate Random Bulk Response with noOfSuccessItems as BulkItemResponse, and include BulkItemResponse.Failure with + // random error message, if hasFailures is true. + public static BulkResponse generateRandomBulkResponse(int noOfSuccessItems, boolean hasFailures) { + long took = randomNonNegativeLong(); + long ingestTook = randomNonNegativeLong(); + if (noOfSuccessItems < 1) { + return new BulkResponse(null, took, ingestTook); + } + List items = new ArrayList<>(); + IntStream.range(0, noOfSuccessItems) + .forEach(shardId -> items.add(new BulkItemResponse(shardId, DocWriteRequest.OpType.CREATE, randomIndexResponse()))); + if (hasFailures) { + final BulkItemResponse.Failure failedToIndex = new BulkItemResponse.Failure( + randomLowerCaseString(), + randomLowerCaseString(), + new OpenSearchException(randomLowerCaseString()) + ); + items.add(new BulkItemResponse(randomIntBetween(0, MAX_SHARD_ID), DocWriteRequest.OpType.CREATE, failedToIndex)); + } + return new BulkResponse(items.toArray(BulkItemResponse[]::new), took, ingestTook); + } + + public static StringBuilder buildFieldNameValuePair(Object field, Object value) { + StringBuilder builder = new StringBuilder(); + builder.append("\"").append(field).append("\":"); + if (!(value instanceof String)) { + return builder.append(value); + } + return builder.append("\"").append(value).append("\""); + } + +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadataTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadataTests.java new file mode 100644 index 000000000..fc229c2e8 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadataTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.common; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URLConnection; + +import org.opensearch.common.SuppressForbidden; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; + +@SuppressForbidden(reason = "unit test") +public class TIFMetadataTests extends SecurityAnalyticsRestTestCase { + + public void testInternalBuild_whenCalled_thenCorrectUserAgentValueIsSet() throws IOException { + URLConnection connection = mock(URLConnection.class); + File manifestFile = new File(this.getClass().getClassLoader().getResource("threatIntel/manifest.json").getFile()); + when(connection.getInputStream()).thenReturn(new FileInputStream(manifestFile)); + + // Run + TIFMetadata manifest = TIFMetadata.Builder.internalBuild(connection); + + // Verify + verify(connection).addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + assertEquals("https://test.com/db.zip", manifest.getUrl()); + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java new file mode 100644 index 000000000..d9390af7a --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +import static org.mockito.Mockito.mock; +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.RENEW_AFTER_IN_SECONDS; + +import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Before; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestHelper; + +public class ThreatIntelLockServiceTests extends ThreatIntelTestCase { + private TIFLockService threatIntelLockService; + private TIFLockService noOpsLockService; + + @Before + public void init() { + threatIntelLockService = new TIFLockService(clusterService, verifyingClient); + noOpsLockService = new TIFLockService(clusterService, client); + } + + public void testAcquireLock_whenValidInput_thenSucceed() { + // Cannot test because LockService is final class + // Simply calling method to increase coverage + noOpsLockService.acquireLock(ThreatIntelTestHelper.randomLowerCaseString(), randomPositiveLong(), mock(ActionListener.class)); + } + + public void testAcquireLock_whenCalled_thenNotBlocked() { + long expectedDurationInMillis = 1000; + Instant before = Instant.now(); + assertTrue(threatIntelLockService.acquireLock(null, null).isEmpty()); + Instant after = Instant.now(); + assertTrue(after.toEpochMilli() - before.toEpochMilli() < expectedDurationInMillis); + } + + public void testReleaseLock_whenValidInput_thenSucceed() { + // Cannot test because LockService is final class + // Simply calling method to increase coverage + noOpsLockService.releaseLock(null); + } + + public void testRenewLock_whenCalled_thenNotBlocked() { + long expectedDurationInMillis = 1000; + Instant before = Instant.now(); + assertNull(threatIntelLockService.renewLock(null)); + Instant after = Instant.now(); + assertTrue(after.toEpochMilli() - before.toEpochMilli() < expectedDurationInMillis); + } + + public void testGetRenewLockRunnable_whenLockIsFresh_thenDoNotRenew() { + LockModel lockModel = new LockModel( + ThreatIntelTestHelper.randomLowerCaseString(), + ThreatIntelTestHelper.randomLowerCaseString(), + Instant.now(), + LOCK_DURATION_IN_SECONDS, + false + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof UpdateRequest); + return new UpdateResponse( + mock(ShardId.class), + ThreatIntelTestHelper.randomLowerCaseString(), + randomPositiveLong(), + randomPositiveLong(), + randomPositiveLong(), + DocWriteResponse.Result.UPDATED + ); + }); + + AtomicReference reference = new AtomicReference<>(lockModel); + threatIntelLockService.getRenewLockRunnable(reference).run(); + assertEquals(lockModel, reference.get()); + } + + public void testGetRenewLockRunnable_whenLockIsStale_thenRenew() { + LockModel lockModel = new LockModel( + ThreatIntelTestHelper.randomLowerCaseString(), + ThreatIntelTestHelper.randomLowerCaseString(), + Instant.now().minusSeconds(RENEW_AFTER_IN_SECONDS), + LOCK_DURATION_IN_SECONDS, + false + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof UpdateRequest); + return new UpdateResponse( + mock(ShardId.class), + ThreatIntelTestHelper.randomLowerCaseString(), + randomPositiveLong(), + randomPositiveLong(), + randomPositiveLong(), + DocWriteResponse.Result.UPDATED + ); + }); + + AtomicReference reference = new AtomicReference<>(lockModel); + threatIntelLockService.getRenewLockRunnable(reference).run(); + assertNotEquals(lockModel, reference.get()); + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java new file mode 100644 index 000000000..ab8520286 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.jobscheduler.spi.JobDocVersion; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestHelper; +public class TIFJobExtensionTests extends ThreatIntelTestCase { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + + public void testBasic() { + TIFJobExtension extension = new TIFJobExtension(); + assertEquals("scheduler_sap_threatintel_job", extension.getJobType()); + assertEquals(JOB_INDEX_NAME, extension.getJobIndex()); + assertEquals(TIFJobRunner.getJobRunnerInstance(), extension.getJobRunner()); + } + + public void testParser() throws Exception { + TIFJobExtension extension = new TIFJobExtension(); + String id = ThreatIntelTestHelper.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + TIFJobParameter tifJobParameter = new TIFJobParameter(id, schedule); + + TIFJobParameter anotherTifJobParameter = (TIFJobParameter) extension.getJobParser() + .parse( + createParser(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), null)), + ThreatIntelTestHelper.randomLowerCaseString(), + new JobDocVersion(randomPositiveLong(), randomPositiveLong(), randomPositiveLong()) + ); + log.info("first"); + log.error(tifJobParameter); + log.error(tifJobParameter.getName()); + log.error(tifJobParameter.getCurrentIndex()); + log.info("second"); + log.error(anotherTifJobParameter); + log.error(anotherTifJobParameter.getName()); + log.error(anotherTifJobParameter.getCurrentIndex()); + + //same values but technically diff indices + + assertTrue(tifJobParameter.equals(anotherTifJobParameter)); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java new file mode 100644 index 000000000..148d16e93 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java @@ -0,0 +1,385 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.List; + +import org.apache.lucene.search.TotalHits; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.get.MultiGetItemResponse; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.get.MultiGetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.cluster.routing.Preference; +import org.opensearch.common.Randomness; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestHelper; + +public class TIFJobParameterServiceTests extends ThreatIntelTestCase { + private TIFJobParameterService tifJobParameterService; + + @Before + public void init() { + tifJobParameterService = new TIFJobParameterService(verifyingClient, clusterService); + } + + public void testCreateIndexIfNotExists_whenIndexExist_thenCreateRequestIsNotCalled() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(true); + + // Verify + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException("Shouldn't get called"); }); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testCreateIndexIfNotExists_whenIndexExist_thenCreateRequestIsCalled() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + + // Verify + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof CreateIndexRequest); + CreateIndexRequest request = (CreateIndexRequest) actionRequest; + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals("1", request.settings().get("index.number_of_shards")); + assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); + assertEquals("true", request.settings().get("index.hidden")); + assertNotNull(request.mappings()); + return null; + }); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testCreateIndexIfNotExists_whenIndexCreatedAlready_thenExceptionIsIgnored() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + verifyingClient.setExecuteVerifier( + (actionResponse, actionRequest) -> { throw new ResourceAlreadyExistsException(TIFJobExtension.JOB_INDEX_NAME); } + ); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testCreateIndexIfNotExists_whenExceptionIsThrown_thenExceptionIsThrown() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException(); }); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + expectThrows(RuntimeException.class, () -> stepListener.result()); + } + + public void testUpdateTIFJobParameter_whenValidInput_thenSucceed() throws Exception { + String tifJobName = ThreatIntelTestHelper.randomLowerCaseString(); + TIFJobParameter tifJobParameter = new TIFJobParameter( + tifJobName, + new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS) + ); + Instant previousTime = Instant.now().minusMillis(1); + tifJobParameter.setLastUpdateTime(previousTime); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest request = (IndexRequest) actionRequest; + assertEquals(tifJobParameter.getName(), request.id()); + assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); + return null; + }); + + tifJobParameterService.updateJobSchedulerParameter(tifJobParameter); + assertTrue(previousTime.isBefore(tifJobParameter.getLastUpdateTime())); + } + + public void testPutTifJobParameter_whenValidInput_thenSucceed() { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + Instant previousTime = Instant.now().minusMillis(1); + tifJobParameter.setLastUpdateTime(previousTime); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest indexRequest = (IndexRequest) actionRequest; + assertEquals(TIFJobExtension.JOB_INDEX_NAME, indexRequest.index()); + assertEquals(tifJobParameter.getName(), indexRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, indexRequest.getRefreshPolicy()); + assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); + return null; + }); + + tifJobParameterService.putTIFJobParameter(tifJobParameter, mock(ActionListener.class)); + assertTrue(previousTime.isBefore(tifJobParameter.getLastUpdateTime())); + } + + public void testGetTifJobParameter_whenException_thenNull() throws Exception { + TIFJobParameter tifJobParameter = setupClientForGetRequest(true, new IndexNotFoundException(TIFJobExtension.JOB_INDEX_NAME)); + assertNull(tifJobParameterService.getJobParameter(tifJobParameter.getName())); + } + + public void testGetTifJobParameter_whenExist_thenReturnTifJobParameter() throws Exception { + TIFJobParameter tifJobParameter = setupClientForGetRequest(true, null); + assertEquals(tifJobParameter, tifJobParameterService.getJobParameter(tifJobParameter.getName())); + } + + public void testGetTifJobParameter_whenNotExist_thenNull() throws Exception { + TIFJobParameter tifJobParameter = setupClientForGetRequest(false, null); + assertNull(tifJobParameterService.getJobParameter(tifJobParameter.getName())); + } + + public void testGetTifJobParameter_whenExistWithListener_thenListenerIsCalledWithTifJobParameter() { + TIFJobParameter tifJobParameter = setupClientForGetRequest(true, null); + ActionListener listener = mock(ActionListener.class); + tifJobParameterService.getJobParameter(tifJobParameter.getName(), listener); + verify(listener).onResponse(eq(tifJobParameter)); + } + + public void testGetTifJobParameter_whenNotExistWithListener_thenListenerIsCalledWithNull() { + TIFJobParameter tifJobParameter = setupClientForGetRequest(false, null); + ActionListener listener = mock(ActionListener.class); + tifJobParameterService.getJobParameter(tifJobParameter.getName(), listener); + verify(listener).onResponse(null); + } + + private TIFJobParameter setupClientForGetRequest(final boolean isExist, final RuntimeException exception) { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof GetRequest); + GetRequest request = (GetRequest) actionRequest; + assertEquals(tifJobParameter.getName(), request.id()); + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + GetResponse response = getMockedGetResponse(isExist ? tifJobParameter : null); + if (exception != null) { + throw exception; + } + return response; + }); + return tifJobParameter; + } + + public void testDeleteTifJobParameter_whenValidInput_thenSucceed() { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof DeleteRequest); + DeleteRequest request = (DeleteRequest) actionRequest; + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(DocWriteRequest.OpType.DELETE, request.opType()); + assertEquals(tifJobParameter.getName(), request.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); + + DeleteResponse response = mock(DeleteResponse.class); + when(response.status()).thenReturn(RestStatus.OK); + return response; + }); + + // Run + tifJobParameterService.deleteTIFJobParameter(tifJobParameter); + } + + public void testDeleteTifJobParameter_whenIndexNotFound_thenThrowException() { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + DeleteResponse response = mock(DeleteResponse.class); + when(response.status()).thenReturn(RestStatus.NOT_FOUND); + return response; + }); + + // Run + expectThrows(ResourceNotFoundException.class, () -> tifJobParameterService.deleteTIFJobParameter(tifJobParameter)); + } + + public void testGetTifJobParameter_whenValidInput_thenSucceed() { + List tifJobParameters = Arrays.asList(randomTifJobParameter(), randomTifJobParameter()); + String[] names = tifJobParameters.stream().map(TIFJobParameter::getName).toArray(String[]::new); + ActionListener> listener = mock(ActionListener.class); + MultiGetItemResponse[] multiGetItemResponses = tifJobParameters.stream().map(tifJobParameter -> { + GetResponse getResponse = getMockedGetResponse(tifJobParameter); + MultiGetItemResponse multiGetItemResponse = mock(MultiGetItemResponse.class); + when(multiGetItemResponse.getResponse()).thenReturn(getResponse); + return multiGetItemResponse; + }).toArray(MultiGetItemResponse[]::new); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof MultiGetRequest); + MultiGetRequest request = (MultiGetRequest) actionRequest; + assertEquals(2, request.getItems().size()); + for (MultiGetRequest.Item item : request.getItems()) { + assertEquals(TIFJobExtension.JOB_INDEX_NAME, item.index()); + assertTrue(tifJobParameters.stream().filter(tifJobParameter -> tifJobParameter.getName().equals(item.id())).findAny().isPresent()); + } + + MultiGetResponse response = mock(MultiGetResponse.class); + when(response.getResponses()).thenReturn(multiGetItemResponses); + return response; + }); + + // Run + tifJobParameterService.getTIFJobParameters(names, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(listener).onResponse(captor.capture()); + assertEquals(tifJobParameters, captor.getValue()); + + } + + public void testGetAllTifJobParameter_whenAsynchronous_thenSuccee() { + List tifJobParameters = Arrays.asList(randomTifJobParameter(), randomTifJobParameter()); + ActionListener> listener = mock(ActionListener.class); + SearchHits searchHits = getMockedSearchHits(tifJobParameters); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof SearchRequest); + SearchRequest request = (SearchRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.indices()[0]); + assertEquals(QueryBuilders.matchAllQuery(), request.source().query()); + assertEquals(1000, request.source().size()); + assertEquals(Preference.PRIMARY.type(), request.preference()); + + SearchResponse response = mock(SearchResponse.class); + when(response.getHits()).thenReturn(searchHits); + return response; + }); + + // Run + tifJobParameterService.getAllTIFJobParameters(listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(listener).onResponse(captor.capture()); + assertEquals(tifJobParameters, captor.getValue()); + } + + public void testGetAllTifJobParameter_whenSynchronous_thenSucceed() { + List tifJobParameters = Arrays.asList(randomTifJobParameter(), randomTifJobParameter()); + SearchHits searchHits = getMockedSearchHits(tifJobParameters); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof SearchRequest); + SearchRequest request = (SearchRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.indices()[0]); + assertEquals(QueryBuilders.matchAllQuery(), request.source().query()); + assertEquals(1000, request.source().size()); + assertEquals(Preference.PRIMARY.type(), request.preference()); + + SearchResponse response = mock(SearchResponse.class); + when(response.getHits()).thenReturn(searchHits); + return response; + }); + + // Run + tifJobParameterService.getAllTIFJobParameters(); + + // Verify + assertEquals(tifJobParameters, tifJobParameterService.getAllTIFJobParameters()); + } + + public void testUpdateTifJobParameter_whenValidInput_thenUpdate() { + List tifJobParameters = Arrays.asList(randomTifJobParameter(), randomTifJobParameter()); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof BulkRequest); + BulkRequest bulkRequest = (BulkRequest) actionRequest; + assertEquals(2, bulkRequest.requests().size()); + for (int i = 0; i < bulkRequest.requests().size(); i++) { + IndexRequest request = (IndexRequest) bulkRequest.requests().get(i); + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(tifJobParameters.get(i).getName(), request.id()); + assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); + } + return null; + }); + + tifJobParameterService.updateJobSchedulerParameter(tifJobParameters, mock(ActionListener.class)); + } + + private SearchHits getMockedSearchHits(List tifJobParameters) { + SearchHit[] searchHitArray = tifJobParameters.stream().map(this::toBytesReference).map(this::toSearchHit).toArray(SearchHit[]::new); + + return new SearchHits(searchHitArray, new TotalHits(1l, TotalHits.Relation.EQUAL_TO), 1); + } + + private GetResponse getMockedGetResponse(TIFJobParameter tifJobParameter) { + GetResponse response = mock(GetResponse.class); + when(response.isExists()).thenReturn(tifJobParameter != null); + when(response.getSourceAsBytesRef()).thenReturn(toBytesReference(tifJobParameter)); + return response; + } + + private BytesReference toBytesReference(TIFJobParameter tifJobParameter) { + if (tifJobParameter == null) { + return null; + } + + try { + return BytesReference.bytes(tifJobParameter.toXContent(JsonXContent.contentBuilder(), null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private SearchHit toSearchHit(BytesReference bytesReference) { + SearchHit searchHit = new SearchHit(Randomness.get().nextInt()); + searchHit.sourceRef(bytesReference); + return searchHit; + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java new file mode 100644 index 000000000..90a67f74b --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestHelper; + +public class TIFJobParameterTests extends ThreatIntelTestCase { + private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + + public void testParser_whenAllValueIsFilled_thenSucceed() throws IOException { // TODO: same issue + String id = ThreatIntelTestHelper.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + TIFJobParameter tifJobParameter = new TIFJobParameter(id, schedule); + tifJobParameter.enable(); + tifJobParameter.setCurrentIndex(ThreatIntelTestHelper.randomLowerCaseString()); + tifJobParameter.getUpdateStats().setLastProcessingTimeInMillis(randomPositiveLong()); + tifJobParameter.getUpdateStats().setLastSucceededAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + tifJobParameter.getUpdateStats().setLastSkippedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + tifJobParameter.getUpdateStats().setLastFailedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + + TIFJobParameter anotherTIFJobParameter = TIFJobParameter.PARSER.parse( + createParser(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), null)), + null + ); + + log.info("first"); + log.error(tifJobParameter); + log.error(tifJobParameter.getName()); + log.error(tifJobParameter.getCurrentIndex()); + log.info("second"); + log.error(anotherTIFJobParameter); + log.error(anotherTIFJobParameter.getName()); + log.error(anotherTIFJobParameter.getCurrentIndex()); + + assertTrue(tifJobParameter.equals(anotherTIFJobParameter)); + } + + public void testParser_whenNullForOptionalFields_thenSucceed() throws IOException { // TODO: same issue + String id = ThreatIntelTestHelper.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + TIFJobParameter datasource = new TIFJobParameter(id, schedule); + TIFJobParameter anotherDatasource = TIFJobParameter.PARSER.parse( + createParser(datasource.toXContent(XContentFactory.jsonBuilder(), null)), + null + ); + assertTrue(datasource.equals(anotherDatasource)); + } + + public void testCurrentIndexName_whenNotExpired_thenReturnName() { + String id = ThreatIntelTestHelper.randomLowerCaseString(); + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setName(id); + datasource.setCurrentIndex(datasource.newIndexName(ThreatIntelTestHelper.randomLowerCaseString())); + + assertNotNull(datasource.currentIndexName()); + } + + public void testNewIndexName_whenCalled_thenReturnedExpectedValue() { + String name = ThreatIntelTestHelper.randomLowerCaseString(); + String suffix = ThreatIntelTestHelper.randomLowerCaseString(); + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setName(name); + assertEquals(String.format(Locale.ROOT, "%s.%s.%s", THREAT_INTEL_DATA_INDEX_NAME_PREFIX, name, suffix), datasource.newIndexName(suffix)); + } + + public void testLockDurationSeconds() { + TIFJobParameter datasource = new TIFJobParameter(); + assertNotNull(datasource.getLockDurationSeconds()); + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java new file mode 100644 index 000000000..e30f2ecfc --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java @@ -0,0 +1,177 @@ + +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.internal.verification.VerificationModeFactory.times; + +import java.io.IOException; +import java.time.Instant; +import java.util.Optional; + +import org.junit.Before; + +import org.opensearch.jobscheduler.spi.JobDocVersion; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestHelper; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; + +public class TIFJobRunnerTests extends ThreatIntelTestCase { + @Before + public void init() { + TIFJobRunner.getJobRunnerInstance() + .initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelExecutor, threatIntelLockService, threadPool); + } + + public void testGetJobRunnerInstance_whenCalledAgain_thenReturnSameInstance() { + assertTrue(TIFJobRunner.getJobRunnerInstance() == TIFJobRunner.getJobRunnerInstance()); + } + + public void testRunJob_whenInvalidClass_thenThrowException() { + JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); + String jobIndexName = ThreatIntelTestHelper.randomLowerCaseString(); + String jobId = ThreatIntelTestHelper.randomLowerCaseString(); + JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + + // Run + expectThrows(IllegalStateException.class, () -> TIFJobRunner.getJobRunnerInstance().runJob(jobParameter, jobExecutionContext)); + } + + public void testRunJob_whenValidInput_thenSucceed() throws IOException { + JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); + String jobIndexName = ThreatIntelTestHelper.randomLowerCaseString(); + String jobId = ThreatIntelTestHelper.randomLowerCaseString(); + JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); + TIFJobParameter tifJobParameter = randomTifJobParameter(); + + LockModel lockModel = randomLockModel(); + when(threatIntelLockService.acquireLock(tifJobParameter.getName(), TIFLockService.LOCK_DURATION_IN_SECONDS)).thenReturn( + Optional.of(lockModel) + ); + + // Run + TIFJobRunner.getJobRunnerInstance().runJob(tifJobParameter, jobExecutionContext); + + // Verify + verify(threatIntelLockService).acquireLock(tifJobParameter.getName(), threatIntelLockService.LOCK_DURATION_IN_SECONDS); + verify(tifJobParameterService).getJobParameter(tifJobParameter.getName()); + verify(threatIntelLockService).releaseLock(lockModel); + } + + public void testUpdateDatasourceRunner_whenExceptionBeforeAcquiringLock_thenNoReleaseLock() { + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + when(jobParameter.getName()).thenReturn(ThreatIntelTestHelper.randomLowerCaseString()); + when(threatIntelLockService.acquireLock(jobParameter.getName(), TIFLockService.LOCK_DURATION_IN_SECONDS)).thenThrow( + new RuntimeException() + ); + + // Run + expectThrows(Exception.class, () -> TIFJobRunner.getJobRunnerInstance().updateJobRunner(jobParameter).run()); + + // Verify + verify(threatIntelLockService, never()).releaseLock(any()); + } + + public void testUpdateDatasourceRunner_whenExceptionAfterAcquiringLock_thenReleaseLock() throws IOException { + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + when(jobParameter.getName()).thenReturn(ThreatIntelTestHelper.randomLowerCaseString()); + LockModel lockModel = randomLockModel(); + when(threatIntelLockService.acquireLock(jobParameter.getName(), TIFLockService.LOCK_DURATION_IN_SECONDS)).thenReturn( + Optional.of(lockModel) + ); + when(tifJobParameterService.getJobParameter(jobParameter.getName())).thenThrow(new RuntimeException()); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobRunner(jobParameter).run(); + + // Verify + verify(threatIntelLockService).releaseLock(any()); + } + + public void testUpdateDatasource_whenDatasourceDoesNotExist_thenDoNothing() throws IOException { + TIFJobParameter datasource = new TIFJobParameter(); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(datasource, mock(Runnable.class)); + + // Verify + verify(tifJobUpdateService, never()).deleteAllTifdIndices(any()); + } + + public void testUpdateDatasource_whenInvalidState_thenUpdateLastFailedAt() throws IOException { + TIFJobParameter datasource = new TIFJobParameter(); + datasource.enable(); + datasource.getUpdateStats().setLastFailedAt(null); + datasource.setState(randomStateExcept(TIFJobState.AVAILABLE)); + when(tifJobParameterService.getJobParameter(datasource.getName())).thenReturn(datasource); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(datasource, mock(Runnable.class)); + + // Verify + assertFalse(datasource.isEnabled()); + assertNotNull(datasource.getUpdateStats().getLastFailedAt()); + verify(tifJobParameterService).updateJobSchedulerParameter(datasource); + } + + public void testUpdateDatasource_whenValidInput_thenSucceed() throws IOException { + TIFJobParameter datasource = randomTifJobParameter(); + datasource.setState(TIFJobState.AVAILABLE); + when(tifJobParameterService.getJobParameter(datasource.getName())).thenReturn(datasource); + Runnable renewLock = mock(Runnable.class); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(datasource, renewLock); + + // Verify + verify(tifJobUpdateService, times(1)).deleteAllTifdIndices(datasource); + verify(tifJobUpdateService).createThreatIntelFeedData(datasource, renewLock); + verify(tifJobUpdateService).updateJobSchedulerParameter(datasource, datasource.getSchedule(), TIFJobTask.ALL); + } + + public void testUpdateDatasource_whenDeleteTask_thenDeleteOnly() throws IOException { + TIFJobParameter datasource = randomTifJobParameter(); + datasource.setState(TIFJobState.AVAILABLE); + datasource.setTask(TIFJobTask.DELETE_UNUSED_INDICES); + when(tifJobParameterService.getJobParameter(datasource.getName())).thenReturn(datasource); + Runnable renewLock = mock(Runnable.class); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(datasource, renewLock); + + // Verify + verify(tifJobUpdateService, times(1)).deleteAllTifdIndices(datasource); + verify(tifJobUpdateService, never()).createThreatIntelFeedData(datasource, renewLock); + verify(tifJobUpdateService).updateJobSchedulerParameter(datasource, datasource.getSchedule(), TIFJobTask.ALL); + } + + public void testUpdateDatasourceExceptionHandling() throws IOException { + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setName(ThreatIntelTestHelper.randomLowerCaseString()); + datasource.getUpdateStats().setLastFailedAt(null); + when(tifJobParameterService.getJobParameter(datasource.getName())).thenReturn(datasource); + doThrow(new RuntimeException("test failure")).when(tifJobUpdateService).deleteAllTifdIndices(any()); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(datasource, mock(Runnable.class)); + + // Verify + assertNotNull(datasource.getUpdateStats().getLastFailedAt()); + verify(tifJobParameterService).updateJobSchedulerParameter(datasource); + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateServiceTests.java new file mode 100644 index 000000000..06f635a34 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateServiceTests.java @@ -0,0 +1,205 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.junit.Before; +import org.opensearch.OpenSearchException; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedParser; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestHelper; +import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; + + +@SuppressForbidden(reason = "unit test") +public class TIFJobUpdateServiceTests extends ThreatIntelTestCase { + private TIFJobUpdateService datasourceUpdateService; + + @Before + public void init() { + datasourceUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService); + } + + public void testUpdateOrCreateThreatIntelFeedData_whenHashValueIsSame_thenSkipUpdate() throws IOException { + List containedIocs = new ArrayList<>(); + containedIocs.add("ip"); + TIFMetadata tifMetadata = new TIFMetadata("id", "url", "name", "org", "desc", "type", containedIocs, "0"); + + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setState(TIFJobState.AVAILABLE); + + // Run + datasourceUpdateService.createThreatIntelFeedData(datasource, mock(Runnable.class)); + + // Verify + assertNotNull(datasource.getUpdateStats().getLastSkippedAt()); + verify(tifJobParameterService).updateJobSchedulerParameter(datasource); + } + + public void testUpdateOrCreateThreatIntelFeedData_whenInvalidData_thenThrowException() throws IOException { + List containedIocs = new ArrayList<>(); + containedIocs.add("ip"); + TIFMetadata tifMetadata = new TIFMetadata("id", "url", "name", "org", "desc", "type", containedIocs, "0"); + + File sampleFile = new File( + this.getClass().getClassLoader().getResource("threatIntel/sample_invalid_less_than_two_fields.csv").getFile() + ); + when(ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setState(TIFJobState.AVAILABLE); + // Run + expectThrows(OpenSearchException.class, () -> datasourceUpdateService.createThreatIntelFeedData(datasource, mock(Runnable.class))); + } + + public void testUpdateOrCreateThreatIntelFeedData_whenIncompatibleFields_thenThrowException() throws IOException { + List containedIocs = new ArrayList<>(); + containedIocs.add("ip"); + TIFMetadata tifMetadata = new TIFMetadata("id", "https://feodotracker.abuse.ch/downloads/ipblocklist_aggressive.csv", "name", "org", "desc", "type", containedIocs, "0"); + + File sampleFile = new File(this.getClass().getClassLoader().getResource("threatIntel/sample_valid.csv").getFile()); + when(ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setState(TIFJobState.AVAILABLE); + + + // Run + expectThrows(OpenSearchException.class, () -> datasourceUpdateService.createThreatIntelFeedData(datasource, mock(Runnable.class))); + } + + public void testUpdateOrCreateThreatIntelFeedData_whenValidInput_thenSucceed() throws IOException { + List containedIocs = new ArrayList<>(); + containedIocs.add("ip"); + TIFMetadata tifMetadata = new TIFMetadata("id", "url", "name", "org", "desc", "type", containedIocs, "0"); + + File sampleFile = new File(this.getClass().getClassLoader().getResource("threatIntel/sample_valid.csv").getFile()); + when(ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.started()).thenReturn(true); + when(routingTable.allShards(anyString())).thenReturn(Arrays.asList(shardRouting)); + + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setState(TIFJobState.AVAILABLE); + + datasource.getUpdateStats().setLastSucceededAt(null); + datasource.getUpdateStats().setLastProcessingTimeInMillis(null); + + // Run + datasourceUpdateService.createThreatIntelFeedData(datasource, mock(Runnable.class)); + + // Verify + + assertNotNull(datasource.getUpdateStats().getLastSucceededAt()); + assertNotNull(datasource.getUpdateStats().getLastProcessingTimeInMillis()); + verify(tifJobParameterService, times(2)).updateJobSchedulerParameter(datasource); + verify(threatIntelFeedDataService).saveThreatIntelFeedDataCSV(eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), any(Runnable.class), tifMetadata); + } + + public void testWaitUntilAllShardsStarted_whenTimedOut_thenThrowException() { + String indexName = ThreatIntelTestHelper.randomLowerCaseString(); + ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.started()).thenReturn(false); + when(routingTable.allShards(indexName)).thenReturn(Arrays.asList(shardRouting)); + + // Run + Exception e = expectThrows(OpenSearchException.class, () -> datasourceUpdateService.waitUntilAllShardsStarted(indexName, 10)); + + // Verify + assertTrue(e.getMessage().contains("did not complete")); + } + + public void testWaitUntilAllShardsStarted_whenInterrupted_thenThrowException() { + String indexName = ThreatIntelTestHelper.randomLowerCaseString(); + ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.started()).thenReturn(false); + when(routingTable.allShards(indexName)).thenReturn(Arrays.asList(shardRouting)); + + // Run + Thread.currentThread().interrupt(); + Exception e = expectThrows(RuntimeException.class, () -> datasourceUpdateService.waitUntilAllShardsStarted(indexName, 10)); + + // Verify + assertEquals(InterruptedException.class, e.getCause().getClass()); + } + + public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { + String datasourceName = ThreatIntelTestHelper.randomLowerCaseString(); + String indexPrefix = String.format(".threatintel-data.%s.", datasourceName); + Instant now = Instant.now(); + String currentIndex = indexPrefix + now.toEpochMilli(); + String oldIndex = indexPrefix + now.minusMillis(1).toEpochMilli(); + String lingeringIndex = indexPrefix + now.minusMillis(2).toEpochMilli(); + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setName(datasourceName); + datasource.setCurrentIndex(currentIndex); + datasource.getIndices().add(currentIndex); + datasource.getIndices().add(oldIndex); + datasource.getIndices().add(lingeringIndex); + + when(metadata.hasIndex(currentIndex)).thenReturn(true); + when(metadata.hasIndex(oldIndex)).thenReturn(true); + when(metadata.hasIndex(lingeringIndex)).thenReturn(false); + + datasourceUpdateService.deleteAllTifdIndices(datasource); + + assertEquals(0, datasource.getIndices().size()); +// assertEquals(currentIndex, datasource.getIndices().get(0)); //TODO: check this + verify(tifJobParameterService).updateJobSchedulerParameter(datasource); + verify(threatIntelFeedDataService).deleteThreatIntelDataIndex(oldIndex); + } + + public void testUpdateDatasource_whenNoChange_thenNoUpdate() { + TIFJobParameter datasource = randomTifJobParameter(); + + // Run + datasourceUpdateService.updateJobSchedulerParameter(datasource, datasource.getSchedule(), datasource.getTask()); + + // Verify + verify(tifJobParameterService, never()).updateJobSchedulerParameter(any()); + } + + public void testUpdateDatasource_whenChange_thenUpdate() { + TIFJobParameter datasource = randomTifJobParameter(); + datasource.setTask(TIFJobTask.ALL); + + // Run + datasourceUpdateService.updateJobSchedulerParameter( + datasource, + new IntervalSchedule(Instant.now(), datasource.getSchedule().getInterval() + 1, ChronoUnit.DAYS), + datasource.getTask() + ); + datasourceUpdateService.updateJobSchedulerParameter(datasource, datasource.getSchedule(), TIFJobTask.DELETE_UNUSED_INDICES); + + // Verify + verify(tifJobParameterService, times(2)).updateJobSchedulerParameter(any()); + } +} diff --git a/src/test/resources/threatIntel/sample_invalid_less_than_two_fields.csv b/src/test/resources/threatIntel/sample_invalid_less_than_two_fields.csv new file mode 100644 index 000000000..08670061c --- /dev/null +++ b/src/test/resources/threatIntel/sample_invalid_less_than_two_fields.csv @@ -0,0 +1,2 @@ +network +1.0.0.0/24 \ No newline at end of file diff --git a/src/test/resources/threatIntel/sample_valid.csv b/src/test/resources/threatIntel/sample_valid.csv new file mode 100644 index 000000000..fad1eb6fd --- /dev/null +++ b/src/test/resources/threatIntel/sample_valid.csv @@ -0,0 +1,3 @@ +ip,region +1.0.0.0/24,Australia +10.0.0.0/24,USA \ No newline at end of file