Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.17] Support index.auto_expand_replicas 0-all for .plugins-ml-config #3019

Merged
merged 1 commit into from
Oct 1, 2024
Merged

[Backport 2.17] Support index.auto_expand_replicas 0-all for .plugins-ml-config #3019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion common/src/main/java/org/opensearch/ml/common/CommonValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public class CommonValue {
public static final Integer ML_TASK_INDEX_SCHEMA_VERSION = 3;
public static final Integer ML_CONNECTOR_SCHEMA_VERSION = 3;
public static final String ML_CONFIG_INDEX = ".plugins-ml-config";
public static final Integer ML_CONFIG_INDEX_SCHEMA_VERSION = 3;
public static final Integer ML_CONFIG_INDEX_SCHEMA_VERSION = 4;
public static final String ML_CONTROLLER_INDEX = ".plugins-ml-controller";
public static final Integer ML_CONTROLLER_INDEX_SCHEMA_VERSION = 1;
public static final String ML_MAP_RESPONSE_KEY = "response";
Expand Down
23 changes: 19 additions & 4 deletions common/src/main/java/org/opensearch/ml/common/utils/IndexUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,24 @@
@Log4j2
public class IndexUtils {

// This setting is for index creation.
public static final Map<String, Object> INDEX_SETTINGS = Map.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-1");
/**
* Default settings for index creation with a single shard and one replica.
* - Sets the number of shards to 1 for better performance in small indices.
* - Uses auto-expand replicas (0-1) to ensure high availability while minimizing resource usage.
*/
public static final Map<String, Object> DEFAULT_INDEX_SETTINGS = Map
.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-1");
/**
* Default settings for index creation with replicas on all nodes.
* - Sets the number of shards to 1 for better performance in small indices.
* - Uses auto-expand replicas (0-all) to ensure a replica on every node, maximizing availability.
* - Caution: This can significantly increase storage requirements and indexing load.
* - Suitable for small, critical indices where maximum redundancy is required.
*/
public static final Map<String, Object> ALL_NODES_REPLICA_INDEX_SETTINGS = Map
.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-all");

// This setting is for index update only, so only dynamic settings should be contained!
public static final Map<String, Object> UPDATED_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-1");
// Note: This does not include static settings like number of shards, which can't be changed after index creation.
public static final Map<String, Object> UPDATED_DEFAULT_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-1");
public static final Map<String, Object> UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-all");
}
22 changes: 18 additions & 4 deletions common/src/test/java/org/opensearch/ml/common/utils/IndexUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,32 @@
public class IndexUtilsTest {

@Test
public void testIndexSettingsContainsExpectedValues() {
Map<String, Object> indexSettings = IndexUtils.INDEX_SETTINGS;
public void testDefaultIndexSettingsContainsExpectedValues() {
Map<String, Object> indexSettings = IndexUtils.DEFAULT_INDEX_SETTINGS;
assertEquals("index.number_of_shards should be 1", indexSettings.get("index.number_of_shards"), "1");
assertEquals("index.auto_expand_replicas should be 0-1", indexSettings.get("index.auto_expand_replicas"), "0-1");
assertEquals("INDEX_SETTINGS should contain exactly 2 settings", 2, indexSettings.size());
}

@Test
public void testUpdatedIndexSettingsContainsExpectedValues() {
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_INDEX_SETTINGS;
public void testAllNodesReplicaIndexSettingsContainsExpectedValues() {
Map<String, Object> indexSettings = IndexUtils.ALL_NODES_REPLICA_INDEX_SETTINGS;
assertEquals("index.number_of_shards should be 1", indexSettings.get("index.number_of_shards"), "1");
assertEquals("index.auto_expand_replicas should be 0-all", indexSettings.get("index.auto_expand_replicas"), "0-all");
assertEquals("INDEX_SETTINGS should contain exactly 2 settings", 2, indexSettings.size());
}

@Test
public void testUpdatedDefaultIndexSettingsContainsExpectedValues() {
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_DEFAULT_INDEX_SETTINGS;
assertEquals("index.auto_expand_replicas should be 0-1", updatedIndexSettings.get("index.auto_expand_replicas"), "0-1");
assertEquals("INDEX_SETTINGS should contain exactly 1 settings", 1, updatedIndexSettings.size());
}

@Test
public void testUpdatedAllNodesReplicaIndexSettingsContainsExpectedValues() {
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS;
assertEquals("index.auto_expand_replicas should be 0-all", updatedIndexSettings.get("index.auto_expand_replicas"), "0-all");
assertEquals("INDEX_SETTINGS should contain exactly 1 settings", 1, updatedIndexSettings.size());
}
}
4 changes: 2 additions & 2 deletions memory/src/main/java/org/opensearch/ml/memory/index/ConversationMetaIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.opensearch.ml.memory.index;

import static org.opensearch.ml.common.conversation.ConversationalIndexConstants.META_INDEX_NAME;
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;

import java.io.IOException;
import java.time.Instant;
Expand Down Expand Up @@ -90,7 +90,7 @@ public void initConversationMetaIndexIfAbsent(ActionListener<Boolean> listener)
CreateIndexRequest request = Requests
.createIndexRequest(META_INDEX_NAME)
.mapping(ConversationalIndexConstants.META_MAPPING, XContentType.JSON)
.settings(INDEX_SETTINGS);
.settings(DEFAULT_INDEX_SETTINGS);
try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) {
ActionListener<Boolean> internalListener = ActionListener.runBefore(listener, () -> threadContext.restore());
ActionListener<CreateIndexResponse> al = ActionListener.wrap(createIndexResponse -> {
Expand Down
4 changes: 2 additions & 2 deletions memory/src/main/java/org/opensearch/ml/memory/index/InteractionsIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.opensearch.ml.memory.index;

import static org.opensearch.ml.common.conversation.ConversationalIndexConstants.INTERACTIONS_INDEX_NAME;
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;

import java.io.IOException;
import java.time.Instant;
Expand Down Expand Up @@ -92,7 +92,7 @@ public void initInteractionsIndexIfAbsent(ActionListener<Boolean> listener) {
CreateIndexRequest request = Requests
.createIndexRequest(INTERACTIONS_INDEX_NAME)
.mapping(ConversationalIndexConstants.INTERACTIONS_MAPPINGS, XContentType.JSON)
.settings(INDEX_SETTINGS);
.settings(DEFAULT_INDEX_SETTINGS);
try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) {
ActionListener<Boolean> internalListener = ActionListener.runBefore(listener, () -> threadContext.restore());
ActionListener<CreateIndexResponse> al = ActionListener.wrap(r -> {
Expand Down
18 changes: 14 additions & 4 deletions ml-algorithms/src/main/java/org/opensearch/ml/engine/indices/MLIndicesHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@

import static org.opensearch.ml.common.CommonValue.META;
import static org.opensearch.ml.common.CommonValue.SCHEMA_VERSION_FIELD;
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.ALL_NODES_REPLICA_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_DEFAULT_INDEX_SETTINGS;

import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -108,7 +110,9 @@ public void initMLIndexIfAbsent(MLIndex index, ActionListener<Boolean> listener)
internalListener.onFailure(e);
}
});
CreateIndexRequest request = new CreateIndexRequest(indexName).mapping(mapping, XContentType.JSON).settings(INDEX_SETTINGS);
CreateIndexRequest request = new CreateIndexRequest(indexName)
.mapping(mapping, XContentType.JSON)
.settings(indexName.equals(MLIndex.CONFIG.getIndexName()) ? ALL_NODES_REPLICA_INDEX_SETTINGS : DEFAULT_INDEX_SETTINGS);
client.admin().indices().create(request, actionListener);
} else {
log.debug("index:{} is already created", indexName);
Expand All @@ -124,7 +128,13 @@ public void initMLIndexIfAbsent(MLIndex index, ActionListener<Boolean> listener)
ActionListener.wrap(response -> {
if (response.isAcknowledged()) {
UpdateSettingsRequest updateSettingRequest = new UpdateSettingsRequest();
updateSettingRequest.indices(indexName).settings(UPDATED_INDEX_SETTINGS);
updateSettingRequest
.indices(indexName)
.settings(
indexName.equals(MLIndex.CONFIG.getIndexName())
? UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS
: UPDATED_DEFAULT_INDEX_SETTINGS
);
client
.admin()
.indices()
Expand Down
Loading