diff --git a/docs/changelog/112972.yaml b/docs/changelog/112972.yaml new file mode 100644 index 0000000000000..5332ac13fd13f --- /dev/null +++ b/docs/changelog/112972.yaml @@ -0,0 +1,6 @@ +pr: 112972 +summary: "ILM: Add `total_shards_per_node` setting to searchable snapshot" +area: ILM+SLM +type: enhancement +issues: + - 112261 diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 4ba4782174bef..73a77bef09bde 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -19,7 +19,7 @@ index>> prefixed with `partial-` to the frozen tier. In other phases, the action In the frozen tier, the action will ignore the setting <>, if it was present in the original index, -to account for the difference in the number of nodes between the frozen and the other tiers. +to account for the difference in the number of nodes between the frozen and the other tiers. To set <> for searchable snapshots, set the `total_shards_per_node` option in the frozen phase's `searchable_snapshot` action within the ILM policy. WARNING: Don't include the `searchable_snapshot` action in both the hot and cold @@ -74,6 +74,9 @@ will be performed on the hot nodes. If using a `searchable_snapshot` action in t force merge will be performed on whatever tier the index is *prior* to the `cold` phase (either `hot` or `warm`). +`total_shards_per_node`:: +The maximum number of shards (replicas and primaries) that will be allocated to a single node for the searchable snapshot index. Defaults to unbounded. + [[ilm-searchable-snapshot-ex]] ==== Examples //// diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 55a3391976057..2cc50a85668c7 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -222,6 +222,7 @@ static TransportVersion def(int id) { public static final TransportVersion FAILURE_STORE_STATUS_IN_INDEX_RESPONSE = def(8_746_00_0); public static final TransportVersion ESQL_AGGREGATION_OPERATOR_STATUS_FINISH_NANOS = def(8_747_00_0); public static final TransportVersion ML_TELEMETRY_MEMORY_ADDED = def(8_748_00_0); + public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE = def(8_749_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java index aac4d74144e95..7d045f2950e1b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java @@ -18,11 +18,13 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; +import java.util.ArrayList; import java.util.Objects; import java.util.Optional; @@ -37,17 +39,34 @@ public class MountSnapshotStep extends AsyncRetryDuringSnapshotActionStep { private final String restoredIndexPrefix; private final MountSearchableSnapshotRequest.Storage storageType; + @Nullable + private final Integer totalShardsPerNode; public MountSnapshotStep( StepKey key, StepKey nextStepKey, Client client, String restoredIndexPrefix, - MountSearchableSnapshotRequest.Storage storageType + MountSearchableSnapshotRequest.Storage storageType, + @Nullable Integer totalShardsPerNode ) { super(key, nextStepKey, client); this.restoredIndexPrefix = restoredIndexPrefix; this.storageType = Objects.requireNonNull(storageType, "a storage type must be specified"); + if (totalShardsPerNode != null && totalShardsPerNode < 1) { + throw new IllegalArgumentException("[" + SearchableSnapshotAction.TOTAL_SHARDS_PER_NODE.getPreferredName() + "] must be >= 1"); + } + this.totalShardsPerNode = totalShardsPerNode; + } + + public MountSnapshotStep( + StepKey key, + StepKey nextStepKey, + Client client, + String restoredIndexPrefix, + MountSearchableSnapshotRequest.Storage storageType + ) { + this(key, nextStepKey, client, restoredIndexPrefix, storageType, null); } @Override @@ -63,6 +82,11 @@ public MountSearchableSnapshotRequest.Storage getStorage() { return storageType; } + @Nullable + public Integer getTotalShardsPerNode() { + return totalShardsPerNode; + } + @Override void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener listener) { String indexName = indexMetadata.getIndex().getName(); @@ -140,6 +164,9 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl final Settings.Builder settingsBuilder = Settings.builder(); overrideTierPreference(this.getKey().phase()).ifPresent(override -> settingsBuilder.put(DataTier.TIER_PREFERENCE, override)); + if (totalShardsPerNode != null) { + settingsBuilder.put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), totalShardsPerNode); + } final MountSearchableSnapshotRequest mountSearchableSnapshotRequest = new MountSearchableSnapshotRequest( TimeValue.MAX_VALUE, @@ -148,9 +175,9 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl snapshotName, indexName, settingsBuilder.build(), - ignoredIndexSettings(this.getKey().phase()), + ignoredIndexSettings(), // we'll not wait for the snapshot to complete in this step as the async steps are executed from threads that shouldn't - // perform expensive operations (ie. clusterStateProcessed) + // perform expensive operations (i.e. clusterStateProcessed) false, storageType ); @@ -198,23 +225,27 @@ static Optional overrideTierPreference(String phase) { * setting, the restored index would be captured by the ILM runner and, depending on what ILM execution state was captured at snapshot * time, make it's way forward from _that_ step forward in the ILM policy. We'll re-set this setting on the restored index at a later * step once we restored a deterministic execution state - * - index.routing.allocation.total_shards_per_node: It is likely that frozen tier has fewer nodes than the hot tier. - * Keeping this setting runs the risk that we will not have enough nodes to allocate all the shards in the - * frozen tier and the user does not have any way of fixing this. For this reason, we ignore this setting when moving to frozen. + * - index.routing.allocation.total_shards_per_node: It is likely that frozen tier has fewer nodes than the hot tier. If this setting + * is not specifically set in the frozen tier, keeping this setting runs the risk that we will not have enough nodes to + * allocate all the shards in the frozen tier and the user does not have any way of fixing this. For this reason, we ignore this + * setting when moving to frozen. We do not ignore this setting if it is specifically set in the mount searchable snapshot step + * of frozen tier. */ - static String[] ignoredIndexSettings(String phase) { + String[] ignoredIndexSettings() { + ArrayList ignoredSettings = new ArrayList<>(); + ignoredSettings.add(LifecycleSettings.LIFECYCLE_NAME); // if we are mounting a searchable snapshot in the hot phase, then we should not change the total_shards_per_node setting - if (TimeseriesLifecycleType.FROZEN_PHASE.equals(phase)) { - return new String[] { - LifecycleSettings.LIFECYCLE_NAME, - ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey() }; + // if total_shards_per_node setting is specifically set for the frozen phase and not propagated from previous phase, + // then it should not be ignored + if (TimeseriesLifecycleType.FROZEN_PHASE.equals(this.getKey().phase()) && this.totalShardsPerNode == null) { + ignoredSettings.add(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey()); } - return new String[] { LifecycleSettings.LIFECYCLE_NAME }; + return ignoredSettings.toArray(new String[0]); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), restoredIndexPrefix, storageType); + return Objects.hash(super.hashCode(), restoredIndexPrefix, storageType, totalShardsPerNode); } @Override @@ -228,6 +259,7 @@ public boolean equals(Object obj) { MountSnapshotStep other = (MountSnapshotStep) obj; return super.equals(obj) && Objects.equals(restoredIndexPrefix, other.restoredIndexPrefix) - && Objects.equals(storageType, other.storageType); + && Objects.equals(storageType, other.storageType) + && Objects.equals(totalShardsPerNode, other.totalShardsPerNode); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index 5b9b559b4d957..c06dcc0f083d1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.TransportVersions.ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_SNAPSHOT_NAME_SETTING_KEY; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY; @@ -49,6 +50,7 @@ public class SearchableSnapshotAction implements LifecycleAction { public static final ParseField SNAPSHOT_REPOSITORY = new ParseField("snapshot_repository"); public static final ParseField FORCE_MERGE_INDEX = new ParseField("force_merge_index"); + public static final ParseField TOTAL_SHARDS_PER_NODE = new ParseField("total_shards_per_node"); public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; public static final String CONDITIONAL_SKIP_ACTION_STEP = BranchingStep.NAME + "-check-prerequisites"; public static final String CONDITIONAL_SKIP_GENERATE_AND_CLEAN = BranchingStep.NAME + "-check-existing-snapshot"; @@ -58,12 +60,13 @@ public class SearchableSnapshotAction implements LifecycleAction { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, - a -> new SearchableSnapshotAction((String) a[0], a[1] == null || (boolean) a[1]) + a -> new SearchableSnapshotAction((String) a[0], a[1] == null || (boolean) a[1], (Integer) a[2]) ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), SNAPSHOT_REPOSITORY); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), FORCE_MERGE_INDEX); + PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), TOTAL_SHARDS_PER_NODE); } public static SearchableSnapshotAction parse(XContentParser parser) { @@ -72,22 +75,36 @@ public static SearchableSnapshotAction parse(XContentParser parser) { private final String snapshotRepository; private final boolean forceMergeIndex; + @Nullable + private final Integer totalShardsPerNode; - public SearchableSnapshotAction(String snapshotRepository, boolean forceMergeIndex) { + public SearchableSnapshotAction(String snapshotRepository, boolean forceMergeIndex, @Nullable Integer totalShardsPerNode) { if (Strings.hasText(snapshotRepository) == false) { throw new IllegalArgumentException("the snapshot repository must be specified"); } this.snapshotRepository = snapshotRepository; this.forceMergeIndex = forceMergeIndex; + + if (totalShardsPerNode != null && totalShardsPerNode < 1) { + throw new IllegalArgumentException("[" + TOTAL_SHARDS_PER_NODE.getPreferredName() + "] must be >= 1"); + } + this.totalShardsPerNode = totalShardsPerNode; + } + + public SearchableSnapshotAction(String snapshotRepository, boolean forceMergeIndex) { + this(snapshotRepository, forceMergeIndex, null); } public SearchableSnapshotAction(String snapshotRepository) { - this(snapshotRepository, true); + this(snapshotRepository, true, null); } public SearchableSnapshotAction(StreamInput in) throws IOException { this.snapshotRepository = in.readString(); this.forceMergeIndex = in.readBoolean(); + this.totalShardsPerNode = in.getTransportVersion().onOrAfter(ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE) + ? in.readOptionalInt() + : null; } boolean isForceMergeIndex() { @@ -98,6 +115,10 @@ public String getSnapshotRepository() { return snapshotRepository; } + public Integer getTotalShardsPerNode() { + return totalShardsPerNode; + } + @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { assert false; @@ -298,7 +319,8 @@ public List toSteps(Client client, String phase, StepKey nextStepKey, XPac waitForGreenRestoredIndexKey, client, getRestoredIndexPrefix(mountSnapshotKey), - storageType + storageType, + totalShardsPerNode ); WaitForIndexColorStep waitForGreenIndexHealthStep = new WaitForIndexColorStep( waitForGreenRestoredIndexKey, @@ -402,6 +424,9 @@ public String getWriteableName() { public void writeTo(StreamOutput out) throws IOException { out.writeString(snapshotRepository); out.writeBoolean(forceMergeIndex); + if (out.getTransportVersion().onOrAfter(ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE)) { + out.writeOptionalInt(totalShardsPerNode); + } } @Override @@ -409,6 +434,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(SNAPSHOT_REPOSITORY.getPreferredName(), snapshotRepository); builder.field(FORCE_MERGE_INDEX.getPreferredName(), forceMergeIndex); + if (totalShardsPerNode != null) { + builder.field(TOTAL_SHARDS_PER_NODE.getPreferredName(), totalShardsPerNode); + } builder.endObject(); return builder; } @@ -422,12 +450,14 @@ public boolean equals(Object o) { return false; } SearchableSnapshotAction that = (SearchableSnapshotAction) o; - return Objects.equals(snapshotRepository, that.snapshotRepository) && Objects.equals(forceMergeIndex, that.forceMergeIndex); + return Objects.equals(snapshotRepository, that.snapshotRepository) + && Objects.equals(forceMergeIndex, that.forceMergeIndex) + && Objects.equals(totalShardsPerNode, that.totalShardsPerNode); } @Override public int hashCode() { - return Objects.hash(snapshotRepository, forceMergeIndex); + return Objects.hash(snapshotRepository, forceMergeIndex, totalShardsPerNode); } @Nullable diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 66aa9a24cbcd4..7963d04e0f666 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -224,7 +224,11 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l frozenTime, Collections.singletonMap( SearchableSnapshotAction.NAME, - new SearchableSnapshotAction(randomAlphaOfLength(10), randomBoolean()) + new SearchableSnapshotAction( + randomAlphaOfLength(10), + randomBoolean(), + (randomBoolean() ? null : randomIntBetween(1, 100)) + ) ) ) ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java index 2b5a0535caa0e..8ca7a00ab0948 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java @@ -41,7 +41,8 @@ public MountSnapshotStep createRandomInstance() { StepKey nextStepKey = randomStepKey(); String restoredIndexPrefix = randomAlphaOfLength(10); MountSearchableSnapshotRequest.Storage storage = randomStorageType(); - return new MountSnapshotStep(stepKey, nextStepKey, client, restoredIndexPrefix, storage); + Integer totalShardsPerNode = randomTotalShardsPerNode(true); + return new MountSnapshotStep(stepKey, nextStepKey, client, restoredIndexPrefix, storage, totalShardsPerNode); } public static MountSearchableSnapshotRequest.Storage randomStorageType() { @@ -59,7 +60,8 @@ protected MountSnapshotStep copyInstance(MountSnapshotStep instance) { instance.getNextStepKey(), instance.getClient(), instance.getRestoredIndexPrefix(), - instance.getStorage() + instance.getStorage(), + instance.getTotalShardsPerNode() ); } @@ -69,7 +71,8 @@ public MountSnapshotStep mutateInstance(MountSnapshotStep instance) { StepKey nextKey = instance.getNextStepKey(); String restoredIndexPrefix = instance.getRestoredIndexPrefix(); MountSearchableSnapshotRequest.Storage storage = instance.getStorage(); - switch (between(0, 3)) { + Integer totalShardsPerNode = instance.getTotalShardsPerNode(); + switch (between(0, 4)) { case 0: key = new StepKey(key.phase(), key.action(), key.name() + randomAlphaOfLength(5)); break; @@ -88,10 +91,30 @@ public MountSnapshotStep mutateInstance(MountSnapshotStep instance) { throw new AssertionError("unknown storage type: " + storage); } break; + case 4: + totalShardsPerNode = totalShardsPerNode == null ? 1 : totalShardsPerNode + randomIntBetween(1, 100); + break; default: throw new AssertionError("Illegal randomisation branch"); } - return new MountSnapshotStep(key, nextKey, instance.getClient(), restoredIndexPrefix, storage); + return new MountSnapshotStep(key, nextKey, instance.getClient(), restoredIndexPrefix, storage, totalShardsPerNode); + } + + public void testCreateWithInvalidTotalShardsPerNode() throws Exception { + int invalidTotalShardsPerNode = randomIntBetween(-100, 0); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new MountSnapshotStep( + randomStepKey(), + randomStepKey(), + client, + RESTORED_INDEX_PREFIX, + randomStorageType(), + invalidTotalShardsPerNode + ) + ); + assertEquals("[total_shards_per_node] must be >= 1", exception.getMessage()); } public void testPerformActionFailure() { @@ -345,7 +368,50 @@ public void testIgnoreTotalShardsPerNodeInFrozenPhase() throws Exception { randomStepKey(), client, RESTORED_INDEX_PREFIX, - randomStorageType() + randomStorageType(), + null + ); + performActionAndWait(step, indexMetadata, clusterState, null); + } + } + + public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { + String indexName = randomAlphaOfLength(10); + String policyName = "test-ilm-policy"; + Map ilmCustom = new HashMap<>(); + String snapshotName = indexName + "-" + policyName; + ilmCustom.put("snapshot_name", snapshotName); + String repository = "repository"; + ilmCustom.put("snapshot_repository", repository); + + IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)); + IndexMetadata indexMetadata = indexMetadataBuilder.build(); + + ClusterState clusterState = ClusterState.builder(emptyClusterState()) + .metadata(Metadata.builder().put(indexMetadata, true).build()) + .build(); + + try (var threadPool = createThreadPool()) { + final var client = getRestoreSnapshotRequestAssertingClient( + threadPool, + repository, + snapshotName, + indexName, + RESTORED_INDEX_PREFIX, + indexName, + new String[] { LifecycleSettings.LIFECYCLE_NAME } + ); + MountSnapshotStep step = new MountSnapshotStep( + new StepKey(TimeseriesLifecycleType.FROZEN_PHASE, randomAlphaOfLength(10), randomAlphaOfLength(10)), + randomStepKey(), + client, + RESTORED_INDEX_PREFIX, + randomStorageType(), + randomTotalShardsPerNode(false) ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -401,4 +467,10 @@ protected void } }; } + + private Integer randomTotalShardsPerNode(boolean nullable) { + Integer randomInt = randomIntBetween(1, 100); + Integer randomIntNullable = (randomBoolean() ? null : randomInt); + return nullable ? randomIntNullable : randomInt; + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java index 193d9abeec91d..ca219fdde3d57 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java @@ -16,6 +16,7 @@ import java.util.List; import static org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction.NAME; +import static org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction.TOTAL_SHARDS_PER_NODE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -97,6 +98,16 @@ public void testPrefixAndStorageTypeDefaults() { ); } + public void testCreateWithInvalidTotalShardsPerNode() { + int invalidTotalShardsPerNode = randomIntBetween(-100, 0); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new SearchableSnapshotAction("test", true, invalidTotalShardsPerNode) + ); + assertEquals("[" + TOTAL_SHARDS_PER_NODE.getPreferredName() + "] must be >= 1", exception.getMessage()); + } + private List expectedStepKeysWithForceMerge(String phase) { return List.of( new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), @@ -160,14 +171,23 @@ protected Writeable.Reader instanceReader() { @Override protected SearchableSnapshotAction mutateInstance(SearchableSnapshotAction instance) { - return switch (randomIntBetween(0, 1)) { + return switch (randomIntBetween(0, 2)) { case 0 -> new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10), instance.isForceMergeIndex()); case 1 -> new SearchableSnapshotAction(instance.getSnapshotRepository(), instance.isForceMergeIndex() == false); + case 2 -> new SearchableSnapshotAction( + instance.getSnapshotRepository(), + instance.isForceMergeIndex(), + instance.getTotalShardsPerNode() == null ? 1 : instance.getTotalShardsPerNode() + randomIntBetween(1, 100) + ); default -> throw new IllegalArgumentException("Invalid mutation branch"); }; } static SearchableSnapshotAction randomInstance() { - return new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + return new SearchableSnapshotAction( + randomAlphaOfLengthBetween(5, 10), + randomBoolean(), + (randomBoolean() ? null : randomIntBetween(1, 100)) + ); } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 0e3d0f1b2ec40..fefeaa95319ed 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -48,6 +48,7 @@ import java.util.concurrent.TimeUnit; import static java.util.Collections.singletonMap; +import static org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createComposableTemplate; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; @@ -921,6 +922,61 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio }, 30, TimeUnit.SECONDS); } + public void testSearchableSnapshotTotalShardsPerNode() throws Exception { + String index = "myindex-" + randomAlphaOfLength(4).toLowerCase(Locale.ROOT); + Integer totalShardsPerNode = 2; + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createPolicy( + client(), + policy, + null, + null, + new Phase( + "cold", + TimeValue.ZERO, + singletonMap(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo, randomBoolean())) + ), + new Phase( + "frozen", + TimeValue.ZERO, + singletonMap(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo, randomBoolean(), totalShardsPerNode)) + ), + null + ); + + createIndex(index, Settings.EMPTY); + ensureGreen(index); + indexDocument(client(), index, true); + + // enable ILM after we indexed a document as otherwise ILM might sometimes run so fast the indexDocument call will fail with + // `index_not_found_exception` + updateIndexSettings(index, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy)); + + // wait for snapshot successfully mounted and ILM execution completed + final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; + assertBusy(() -> { + logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); + assertTrue(indexExists(searchableSnapMountedIndexName)); + }, 30, TimeUnit.SECONDS); + assertBusy(() -> { + triggerStateChange(); + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName); + assertThat(stepKeyForIndex.phase(), is("frozen")); + assertThat(stepKeyForIndex.name(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + + // validate total_shards_per_node setting + Map indexSettings = getIndexSettingsAsMap(searchableSnapMountedIndexName); + assertNotNull("expected total_shards_per_node to exist", indexSettings.get(INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey())); + Integer snapshotTotalShardsPerNode = Integer.valueOf((String) indexSettings.get(INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey())); + assertEquals( + "expected total_shards_per_node to be " + totalShardsPerNode + ", but got: " + snapshotTotalShardsPerNode, + snapshotTotalShardsPerNode, + totalShardsPerNode + ); + } + /** * Cause a bit of cluster activity using an empty reroute call in case the `wait-for-index-colour` ILM step missed the * notification that partial-index is now GREEN.