update number of replicas of system indices to 1-20 and number of pri…

…mary shards for system indices to 1 (#1358) (#1360) Signed-off-by: Subhobrata Dey <[email protected]>
opensearch-project · Oct 17, 2024 · 39519ad · 39519ad
1 parent 43c70f3
commit 39519ad
Show file tree

Hide file tree

Showing 10 changed files with 81 additions and 14 deletions.
diff --git a/.../java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/.../java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java
@@ -558,8 +558,17 @@ private void rolloverIndex(
         request.getCreateIndexRequest().index(pattern)
                 .mapping(map)
                 .settings(isCorrelation?
-                        Settings.builder().put("index.hidden", true).put("index.correlation", true).build():
-                        Settings.builder().put("index.hidden", true).build()
+                        Settings.builder()
+                                .put("index.hidden", true)
+                                .put("index.correlation", true)
+                                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                                .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                                .build():
+                        Settings.builder()
+                                .put("index.hidden", true)
+                                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                                .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                                .build()
                 );
         request.addMaxIndexDocsCondition(docsCondition);
         request.addMaxIndexAgeCondition(ageCondition);

diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
@@ -63,10 +63,10 @@
 import org.opensearch.securityanalytics.model.LogType;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
-import static org.opensearch.action.support.ActiveShardCount.ALL;
 import static org.opensearch.securityanalytics.model.FieldMappingDoc.LOG_TYPES;
 import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA;
-
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
 
 /**
  *
@@ -459,7 +459,8 @@ public void ensureConfigIndexIsInitialized(ActionListener<Void> listener) {
             isConfigIndexInitialized = false;
             Settings indexSettings = Settings.builder()
                     .put("index.hidden", true)
-                    .put("index.auto_expand_replicas", "0-all")
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
                     .build();
 
             CreateIndexRequest createIndexRequest = new CreateIndexRequest();

diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java
@@ -19,6 +19,7 @@
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.Streams;
@@ -49,6 +50,9 @@
 import java.util.Map;
 import java.util.UUID;
 
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
+
 public class STIX2IOCFeedStore implements FeedStore {
     public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-iocs";
     public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*";
@@ -234,7 +238,12 @@ private void initFeedIndex(String feedIndexName, ActionListener<CreateIndexRespo
         if (!clusterService.state().routingTable().hasIndex(newActiveIndex)) {
             var indexRequest = new CreateIndexRequest(feedIndexName)
                     .mapping(iocIndexMapping())
-                    .settings(Settings.builder().put("index.hidden", true).build());
+                    .settings(Settings.builder()
+                            .put("index.hidden", true)
+                            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                            .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                            .build()
+                    );
             client.admin().indices().create(indexRequest, ActionListener.wrap(
                     r -> {
                         log.info("Created system index {}", feedIndexName);

diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java
@@ -10,10 +10,10 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-import static org.opensearch.index.IndexSettings.MAX_TERMS_COUNT_SETTING;
-
 public class SecurityAnalyticsSettings {
     public static final String CORRELATION_INDEX = "index.correlation";
+    public static final int minSystemIndexReplicas = 1;
+    public static final int maxSystemIndexReplicas = 20;
 
     public static Setting<TimeValue> INDEX_TIMEOUT = Setting.positiveTimeSetting("plugins.security_analytics.index_timeout",
             TimeValue.timeValueSeconds(60),

diff --git a/.../java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java b/.../java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java
@@ -14,6 +14,7 @@
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentFactory;
@@ -31,6 +32,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
 import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse;
 
 /**
@@ -247,7 +250,9 @@ public void createIndexIfNotExists(final ActionListener<Void> listener) {
     public abstract String getEntityName();
 
     protected Settings.Builder getIndexSettings() {
-        return Settings.builder().put("index.hidden", true);
+        return Settings.builder().put("index.hidden", true)
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas);
     }
 
     public abstract String getEntityAliasName();

diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java
@@ -7,6 +7,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.admin.indices.alias.Alias;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.admin.indices.create.CreateIndexRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexResponse;
@@ -26,6 +27,9 @@
 import java.nio.charset.Charset;
 import java.util.Objects;
 
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
+
 public class CorrelationIndices {
 
     private static final Logger log = LogManager.getLogger(CorrelationIndices.class);
@@ -55,9 +59,15 @@ public static String correlationMappings() throws IOException {
 
     public void initCorrelationIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         if (!correlationIndexExists()) {
+            Settings indexSettings = Settings.builder()
+                    .put("index.hidden", true)
+                    .put("index.correlation", true)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                    .build();
             CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_HISTORY_INDEX_PATTERN)
                     .mapping(correlationMappings())
-                    .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build());
+                    .settings(indexSettings);
             indexRequest.alias(new Alias(CORRELATION_HISTORY_WRITE_INDEX));
             client.admin().indices().create(indexRequest, actionListener);
         } else {
@@ -67,9 +77,15 @@ public void initCorrelationIndex(ActionListener<CreateIndexResponse> actionListe
 
     public void initCorrelationMetadataIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         if (!correlationMetadataIndexExists()) {
+            Settings indexSettings = Settings.builder()
+                    .put("index.hidden", true)
+                    .put("index.correlation", true)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                    .build();
             CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_METADATA_INDEX)
                     .mapping(correlationMappings())
-                    .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build());
+                    .settings(indexSettings);
             client.admin().indices().create(indexRequest, actionListener);
         } else {
             actionListener.onResponse(new CreateIndexResponse(true, true, CORRELATION_METADATA_INDEX));
@@ -136,6 +152,8 @@ public static String correlationAlertIndexMappings() throws IOException {
     public void initCorrelationAlertIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         Settings correlationAlertSettings = Settings.builder()
                 .put("index.hidden", true)
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
                 .build();
         CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_ALERT_INDEX)
                 .mapping(correlationAlertIndexMappings())

diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.admin.indices.create.CreateIndexRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexResponse;
@@ -23,6 +24,9 @@
 import java.util.Objects;
 import org.opensearch.securityanalytics.model.CorrelationRule;
 
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
+
 public class CorrelationRuleIndices {
     private static final Logger log = LogManager.getLogger(CorrelationRuleIndices.class);
 
@@ -45,9 +49,14 @@ public static String correlationRuleIndexMappings() throws IOException {
 
     public void initCorrelationRuleIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         if (!correlationRuleIndexExists()) {
+            Settings indexSettings = Settings.builder()
+                    .put("index.hidden", true)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                    .build();
             CreateIndexRequest indexRequest = new CreateIndexRequest(CorrelationRule.CORRELATION_RULE_INDEX).mapping(
                 correlationRuleIndexMappings()
-            ).settings(Settings.builder().put("index.hidden", true).build());
+            ).settings(indexSettings);
             client.admin().indices().create(indexRequest, actionListener);
         }
     }

diff --git a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java
@@ -21,6 +21,8 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.Objects;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
 
 public class CustomLogTypeIndices {
 
@@ -42,9 +44,11 @@ public static String customLogTypeMappings() throws IOException {
 
     public void initCustomLogTypeIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         if (!customLogTypeIndexExists()) {
+            // Security Analytics log types index is small. 1 primary shard is enough
             Settings indexSettings = Settings.builder()
                     .put("index.hidden", true)
-                    .put("index.auto_expand_replicas", "0-all")
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
                     .build();
             CreateIndexRequest indexRequest = new CreateIndexRequest(LogTypeService.LOG_TYPE_INDEX)
                     .mapping(customLogTypeMappings())

diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java
@@ -23,6 +23,9 @@
 import java.nio.charset.Charset;
 import java.util.Objects;
 
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
+
 public class DetectorIndices {
 
     private static final Logger log = LogManager.getLogger(DetectorIndices.class);
@@ -45,9 +48,14 @@ public static String detectorMappings() throws IOException {
 
     public void initDetectorIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         if (!detectorIndexExists()) {
+            Settings indexSettings = Settings.builder()
+                    .put("index.hidden", true)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                    .build();
             CreateIndexRequest indexRequest = new CreateIndexRequest(Detector.DETECTORS_INDEX)
                     .mapping(detectorMappings())
-                    .settings(Settings.builder().put("index.hidden", true).build());
+                    .settings(indexSettings);
             client.indices().create(indexRequest, actionListener);
         }
     }

diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java
@@ -65,6 +65,8 @@
 import java.util.stream.Stream;
 
 import static org.opensearch.securityanalytics.model.Detector.NO_VERSION;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas;
 
 public class RuleIndices {
 
@@ -93,6 +95,8 @@ public void initRuleIndex(ActionListener<CreateIndexResponse> actionListener, bo
         if (!ruleIndexExists(isPrepackaged)) {
             Settings indexSettings = Settings.builder()
                     .put("index.hidden", true)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
                     .build();
             CreateIndexRequest indexRequest = new CreateIndexRequest(getRuleIndex(isPrepackaged))
                     .mapping(ruleMappings())