diff --git a/docs/reference/ilm/actions/ilm-shrink.asciidoc b/docs/reference/ilm/actions/ilm-shrink.asciidoc index 8800136abf52b..58d223aa4d09e 100644 --- a/docs/reference/ilm/actions/ilm-shrink.asciidoc +++ b/docs/reference/ilm/actions/ilm-shrink.asciidoc @@ -2,7 +2,7 @@ [[ilm-shrink]] === Shrink -Phases allowed: warm +Phases allowed: hot, warm. Sets an index to <> and shrinks it into a new index with fewer primary shards. @@ -11,9 +11,12 @@ For example, if the name of the source index is _logs_, the name of the shrunken index is _shrink-logs_. The shrink action allocates all primary shards of the index to one node so it -can call the <> to shrink the index. +can call the <> to shrink the index. After shrinking, it swaps aliases that point to the original index to the new shrunken index. +To use the `shrink` action in the `hot` phase, the `rollover` action *must* be present. +If no rollover action is configured, {ilm-init} will reject the policy. + [IMPORTANT] If the shrink action is used on a <>, policy execution waits until the leader index rolls over (or is diff --git a/docs/reference/ilm/ilm-index-lifecycle.asciidoc b/docs/reference/ilm/ilm-index-lifecycle.asciidoc index 8392a30e48e07..a43d979a2ec76 100644 --- a/docs/reference/ilm/ilm-index-lifecycle.asciidoc +++ b/docs/reference/ilm/ilm-index-lifecycle.asciidoc @@ -77,22 +77,24 @@ the rollover criteria, it could be 20 minutes before the rollover is complete. * Hot - <> - <> - - <> - <> + - <> + - <> * Warm - <> - <> - <> - <> + - <> - <> - <> * Cold - <> - <> - <> + - <> - <> - <> * Delete - <> - <> - diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index 329d930650192..492ede06cbd43 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -40,7 +40,7 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String DELETE_PHASE = "delete"; static final List VALID_PHASES = Arrays.asList(HOT_PHASE, WARM_PHASE, COLD_PHASE, DELETE_PHASE); static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, - ForceMergeAction.NAME); + ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, @@ -50,14 +50,13 @@ public class TimeseriesLifecycleType implements LifecycleType { static final Set VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS); static final Set VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS); static final Set VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS); - private static final Map> ALLOWED_ACTIONS = new HashMap<>(); + private static final Map> ALLOWED_ACTIONS = Map.of( + HOT_PHASE, VALID_HOT_ACTIONS, + WARM_PHASE, VALID_WARM_ACTIONS, + COLD_PHASE, VALID_COLD_ACTIONS, + DELETE_PHASE, VALID_DELETE_ACTIONS); - static { - ALLOWED_ACTIONS.put(HOT_PHASE, VALID_HOT_ACTIONS); - ALLOWED_ACTIONS.put(WARM_PHASE, VALID_WARM_ACTIONS); - ALLOWED_ACTIONS.put(COLD_PHASE, VALID_COLD_ACTIONS); - ALLOWED_ACTIONS.put(DELETE_PHASE, VALID_DELETE_ACTIONS); - } + static final Set HOT_ACTIONS_THAT_REQUIRE_ROLLOVER = Sets.newHashSet(ShrinkAction.NAME, ForceMergeAction.NAME); private TimeseriesLifecycleType() { } @@ -157,16 +156,16 @@ public List getOrderedActions(Phase phase) { Map actions = phase.getActions(); switch (phase.getName()) { case HOT_PHASE: - return ORDERED_VALID_HOT_ACTIONS.stream().map(a -> actions.getOrDefault(a, null)) + return ORDERED_VALID_HOT_ACTIONS.stream().map(actions::get) .filter(Objects::nonNull).collect(toList()); case WARM_PHASE: - return ORDERED_VALID_WARM_ACTIONS.stream().map(a -> actions.getOrDefault(a, null)) + return ORDERED_VALID_WARM_ACTIONS.stream().map(actions::get) .filter(Objects::nonNull).collect(toList()); case COLD_PHASE: - return ORDERED_VALID_COLD_ACTIONS.stream().map(a -> actions.getOrDefault(a, null)) + return ORDERED_VALID_COLD_ACTIONS.stream().map(actions::get) .filter(Objects::nonNull).collect(toList()); case DELETE_PHASE: - return ORDERED_VALID_DELETE_ACTIONS.stream().map(a -> actions.getOrDefault(a, null)) + return ORDERED_VALID_DELETE_ACTIONS.stream().map(actions::get) .filter(Objects::nonNull).collect(toList()); default: throw new IllegalArgumentException("lifecycle type[" + TYPE + "] does not support phase[" + phase.getName() + "]"); @@ -177,20 +176,20 @@ public List getOrderedActions(Phase phase) { public String getNextActionName(String currentActionName, Phase phase) { List orderedActionNames; switch (phase.getName()) { - case HOT_PHASE: - orderedActionNames = ORDERED_VALID_HOT_ACTIONS; - break; - case WARM_PHASE: - orderedActionNames = ORDERED_VALID_WARM_ACTIONS; - break; - case COLD_PHASE: - orderedActionNames = ORDERED_VALID_COLD_ACTIONS; - break; - case DELETE_PHASE: - orderedActionNames = ORDERED_VALID_DELETE_ACTIONS; - break; - default: - throw new IllegalArgumentException("lifecycle type [" + TYPE + "] does not support phase [" + phase.getName() + "]"); + case HOT_PHASE: + orderedActionNames = ORDERED_VALID_HOT_ACTIONS; + break; + case WARM_PHASE: + orderedActionNames = ORDERED_VALID_WARM_ACTIONS; + break; + case COLD_PHASE: + orderedActionNames = ORDERED_VALID_COLD_ACTIONS; + break; + case DELETE_PHASE: + orderedActionNames = ORDERED_VALID_DELETE_ACTIONS; + break; + default: + throw new IllegalArgumentException("lifecycle type [" + TYPE + "] does not support phase [" + phase.getName() + "]"); } int index = orderedActionNames.indexOf(currentActionName); @@ -226,17 +225,18 @@ public void validate(Collection phases) { }); }); - // Check for forcemerge in 'hot' without a rollover action - if (phases.stream() + // Check for actions in the hot phase that require a rollover + String invalidHotPhaseActions = phases.stream() // Is there a hot phase .filter(phase -> HOT_PHASE.equals(phase.getName())) - // That contains the 'forcemerge' action - .filter(phase -> phase.getActions().containsKey(ForceMergeAction.NAME)) - // But does *not* contain the 'rollover' action? - .anyMatch(phase -> phase.getActions().containsKey(RolloverAction.NAME) == false)) { - // If there is, throw an exception - throw new IllegalArgumentException("the [" + ForceMergeAction.NAME + - "] action may not be used in the [" + HOT_PHASE + + // that does *not* contain the 'rollover' action + .filter(phase -> phase.getActions().containsKey(RolloverAction.NAME) == false) + // but that does have actions that require a rollover action? + .flatMap(phase -> Sets.intersection(phase.getActions().keySet(), HOT_ACTIONS_THAT_REQUIRE_ROLLOVER).stream()) + .collect(Collectors.joining(", ")); + if (Strings.hasText(invalidHotPhaseActions)) { + throw new IllegalArgumentException("the [" + invalidHotPhaseActions + + "] action(s) may not be used in the [" + HOT_PHASE + "] phase without an accompanying [" + RolloverAction.NAME + "] action"); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 1d6306b11544b..916184f2eefc3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -99,7 +99,7 @@ protected LifecyclePolicy createTestInstance() { public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Nullable String lifecycleName) { List phaseNames = TimeseriesLifecycleType.VALID_PHASES; Map phases = new HashMap<>(phaseNames.size()); - Function> validActions = (phase) -> { + Function> validActions = (phase) -> { switch (phase) { case "hot": return TimeseriesLifecycleType.VALID_HOT_ACTIONS; @@ -112,14 +112,14 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Null default: throw new IllegalArgumentException("invalid phase [" + phase + "]"); }}; - Function randomAction = (action) -> { + Function randomAction = (action) -> { switch (action) { case AllocateAction.NAME: return AllocateActionTests.randomInstance(); case DeleteAction.NAME: return new DeleteAction(); case WaitForSnapshotAction.NAME: - return WaitForSnapshotActionTests.randomInstance(); + return WaitForSnapshotActionTests.randomInstance(); case ForceMergeAction.NAME: return ForceMergeActionTests.randomInstance(); case ReadOnlyAction.NAME: @@ -157,7 +157,7 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l List phaseNames = randomSubsetOf( between(0, TimeseriesLifecycleType.VALID_PHASES.size() - 1), TimeseriesLifecycleType.VALID_PHASES); Map phases = new HashMap<>(phaseNames.size()); - Function> validActions = (phase) -> { + Function> validActions = (phase) -> { switch (phase) { case "hot": return TimeseriesLifecycleType.VALID_HOT_ACTIONS; @@ -170,7 +170,7 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l default: throw new IllegalArgumentException("invalid phase [" + phase + "]"); }}; - Function randomAction = (action) -> { + Function randomAction = (action) -> { switch (action) { case AllocateAction.NAME: return AllocateActionTests.randomInstance(); @@ -204,8 +204,9 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l Map actions = new HashMap<>(); List actionNames = randomSubsetOf(validActions.apply(phase)); - // If the hot phase contains a forcemerge, also make sure to add a rollover, or else the policy will not validate - if (phase.equals(TimeseriesLifecycleType.HOT_PHASE) && actionNames.contains(ForceMergeAction.NAME)) { + // If the hot phase has any actions that require a rollover, then ensure there is one so that the policy will validate + if (phase.equals(TimeseriesLifecycleType.HOT_PHASE) + && actionNames.stream().anyMatch(TimeseriesLifecycleType.HOT_ACTIONS_THAT_REQUIRE_ROLLOVER::contains)) { actionNames.add(RolloverAction.NAME); } @@ -238,16 +239,16 @@ protected LifecyclePolicy mutateInstance(LifecyclePolicy instance) throws IOExce String name = instance.getName(); Map phases = instance.getPhases(); switch (between(0, 1)) { - case 0: - name = name + randomAlphaOfLengthBetween(1, 5); - break; - case 1: - String phaseName = randomValueOtherThanMany(phases::containsKey, () -> randomFrom(TimeseriesLifecycleType.VALID_PHASES)); - phases = new LinkedHashMap<>(phases); - phases.put(phaseName, new Phase(phaseName, TimeValue.timeValueSeconds(randomIntBetween(1, 1000)), Collections.emptyMap())); - break; - default: - throw new AssertionError("Illegal randomisation branch"); + case 0: + name = name + randomAlphaOfLengthBetween(1, 5); + break; + case 1: + String phaseName = randomValueOtherThanMany(phases::containsKey, () -> randomFrom(TimeseriesLifecycleType.VALID_PHASES)); + phases = new LinkedHashMap<>(phases); + phases.put(phaseName, new Phase(phaseName, TimeValue.timeValueSeconds(randomIntBetween(1, 1000)), Collections.emptyMap())); + break; + default: + throw new AssertionError("Illegal randomisation branch"); } return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, name, phases); } @@ -300,7 +301,7 @@ public void testToStepsWithTwoPhases() { MockStep secondActionStep = new MockStep(new StepKey("second_phase", "test2", "test"), PhaseCompleteStep.finalStep("second_phase").getKey()); MockStep secondAfter = new MockStep(new StepKey("first_phase", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME), - secondActionStep.getKey()); + secondActionStep.getKey()); MockStep firstActionAnotherStep = new MockStep(new StepKey("first_phase", "test", "bar"), secondAfter.getKey()); MockStep firstActionStep = new MockStep(new StepKey("first_phase", "test", "foo"), firstActionAnotherStep.getKey()); MockStep firstAfter = new MockStep(new StepKey("new", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME), firstActionStep.getKey()); @@ -352,30 +353,30 @@ public void testIsActionSafe() { assertFalse(policy.isActionSafe(new StepKey("second_phase", MockAction.NAME, randomAlphaOfLength(10)))); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> policy.isActionSafe(new StepKey("non_existant_phase", MockAction.NAME, randomAlphaOfLength(10)))); + () -> policy.isActionSafe(new StepKey("non_existant_phase", MockAction.NAME, randomAlphaOfLength(10)))); assertEquals("Phase [non_existant_phase] does not exist in policy [" + policy.getName() + "]", exception.getMessage()); exception = expectThrows(IllegalArgumentException.class, - () -> policy.isActionSafe(new StepKey("first_phase", "non_existant_action", randomAlphaOfLength(10)))); + () -> policy.isActionSafe(new StepKey("first_phase", "non_existant_action", randomAlphaOfLength(10)))); assertEquals("Action [non_existant_action] in phase [first_phase] does not exist in policy [" + policy.getName() + "]", - exception.getMessage()); + exception.getMessage()); assertTrue(policy.isActionSafe(new StepKey("new", randomAlphaOfLength(10), randomAlphaOfLength(10)))); } public void testValidatePolicyName() { - expectThrows(IllegalArgumentException.class, () -> LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0,10) + - "," + randomAlphaOfLengthBetween(0,10))); - expectThrows(IllegalArgumentException.class, () -> LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0,10) + - " " + randomAlphaOfLengthBetween(0,10))); + expectThrows(IllegalArgumentException.class, () -> LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0, 10) + + "," + randomAlphaOfLengthBetween(0, 10))); + expectThrows(IllegalArgumentException.class, () -> LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0, 10) + + " " + randomAlphaOfLengthBetween(0, 10))); expectThrows(IllegalArgumentException.class, () -> LifecyclePolicy.validatePolicyName("_" + randomAlphaOfLengthBetween(1, 20))); expectThrows(IllegalArgumentException.class, () -> LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(256, 1000))); - LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(1,10) + "_" + randomAlphaOfLengthBetween(0,10)); + LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(1, 10) + "_" + randomAlphaOfLengthBetween(0, 10)); - LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0,10) + "-" + randomAlphaOfLengthBetween(0,10)); - LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0,10) + "+" + randomAlphaOfLengthBetween(0,10)); + LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0, 10) + "-" + randomAlphaOfLengthBetween(0, 10)); + LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(0, 10) + "+" + randomAlphaOfLengthBetween(0, 10)); - LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(1,255)); + LifecyclePolicy.validatePolicyName(randomAlphaOfLengthBetween(1, 255)); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 4be976949561c..29ce111fe7503 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -104,7 +104,7 @@ public void testValidateHotPhase() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> validateHotActions.accept(Arrays.asList(ForceMergeAction.NAME))); assertThat(e.getMessage(), - containsString("the [forcemerge] action may not be used in the [hot] phase without an accompanying [rollover] action")); + containsString("the [forcemerge] action(s) may not be used in the [hot] phase without an accompanying [rollover] action")); } } @@ -407,7 +407,6 @@ public void testGetNextActionName() { assertInvalidAction("hot", AllocateAction.NAME, new String[] { RolloverAction.NAME }); assertInvalidAction("hot", DeleteAction.NAME, new String[] { RolloverAction.NAME }); assertInvalidAction("hot", ReadOnlyAction.NAME, new String[] { RolloverAction.NAME }); - assertInvalidAction("hot", ShrinkAction.NAME, new String[] { RolloverAction.NAME }); // Warm Phase assertNextActionName("warm", SetPriorityAction.NAME, UnfollowAction.NAME, diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 05e1ab93e8cae..31543b704ef09 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -559,6 +559,48 @@ public void testShrinkDuringSnapshot() throws Exception { assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot"))); } + public void testShrinkActionInTheHotPhase() throws Exception { + int numShards = 2; + int expectedFinalShards = 1; + String originalIndex = index + "-000001"; + String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex; + + // add a policy + Map hotActions = Map.of( + RolloverAction.NAME, new RolloverAction(null, null, 1L), + ShrinkAction.NAME, new ShrinkAction(expectedFinalShards)); + Map phases = Map.of( + "hot", new Phase("hot", TimeValue.ZERO, hotActions)); + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); + Request createPolicyRequest = new Request("PUT", "_ilm/policy/" + policy); + createPolicyRequest.setJsonEntity("{ \"policy\":" + Strings.toString(lifecyclePolicy) + "}"); + client().performRequest(createPolicyRequest); + + // and a template + Request createTemplateRequest = new Request("PUT", "_template/" + index); + createTemplateRequest.setJsonEntity("{" + + "\"index_patterns\": [\"" + index + "-*\"], \n" + + " \"settings\": {\n" + + " \"number_of_shards\": " + numShards + ",\n" + + " \"number_of_replicas\": 0,\n" + + " \"index.lifecycle.name\": \"" + policy + "\", \n" + + " \"index.lifecycle.rollover_alias\": \"" + alias + "\"\n" + + " }\n" + + "}"); + client().performRequest(createTemplateRequest); + + // then create the index and index a document to trigger rollover + createIndexWithSettings(client(), originalIndex, alias, Settings.builder(), true); + index(client(), originalIndex, "_id", "foo", "bar"); + + assertBusy(() -> assertTrue(indexExists(shrunkenIndex)), 30, TimeUnit.SECONDS); + assertBusy(() -> assertThat(getStepKeyForIndex(client(), shrunkenIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey()))); + assertBusy(() -> { + Map settings = getOnlyIndexSettings(client(), shrunkenIndex); + assertThat(settings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS), equalTo(String.valueOf(expectedFinalShards))); + }); + } + public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception { int numShards = 2; int expectedFinalShards = 1; diff --git a/x-pack/plugin/ilm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ilm/10_basic.yml b/x-pack/plugin/ilm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ilm/10_basic.yml index 3dcfb685323c0..fb985d27be8d8 100644 --- a/x-pack/plugin/ilm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ilm/10_basic.yml +++ b/x-pack/plugin/ilm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ilm/10_basic.yml @@ -107,7 +107,7 @@ setup: body: settings: index.lifecycle.name: "my_moveable_timeseries_lifecycle" - + - do: ilm.put_lifecycle: policy: "my_timeseries_lifecycle" @@ -238,3 +238,23 @@ setup: } } } + + - do: + catch: bad_request + ilm.put_lifecycle: + policy: "my_invalid_lifecycle" + body: | + { + "policy": { + "phases": { + "hot": { + "min_age": "0s", + "actions": { + "shrink": { + "number_of_shards": 1 + } + } + } + } + } + }