Tests: fix testMoveToStepRereadsPolicy flakiness (#64466) (#64579)

testMoveToStepRereadsPolicy relied on an updated ILM policy that had
a rollover condition that enabled the index to be rolled after one second.

This changes the test to use a `max_doc`:1 condition so it's under the
test's control to trigger the condition.

(cherry picked from commit 73ab35a)
Signed-off-by: Andrei Dan <[email protected]>

Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeseriesMoveToStepIT.java

@@ -31,6 +31,7 @@
 import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy;
 import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex;
 import static org.elasticsearch.xpack.TimeSeriesRestDriver.index;
+import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument;
 import static org.hamcrest.Matchers.containsStringIgnoringCase;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -158,7 +159,6 @@ public void testMoveToInjectedStep() throws Exception {
         }, 30, TimeUnit.SECONDS);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53612")
     public void testMoveToStepRereadsPolicy() throws Exception {
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, TimeValue.timeValueHours(1), null), TimeValue.ZERO);
 
@@ -170,9 +170,9 @@ public void testMoveToStepRereadsPolicy() throws Exception {
             true);
 
         assertBusy(() -> assertThat(getStepKeyForIndex(client(), "test-1"),
-            equalTo(new StepKey("hot", "rollover", "check-rollover-ready"))));
+            equalTo(new StepKey("hot", "rollover", "check-rollover-ready"))), 30, TimeUnit.SECONDS);
 
-        createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, TimeValue.timeValueSeconds(1), null), TimeValue.ZERO);
+        createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L), TimeValue.ZERO);
 
         // Move to the same step, which should re-read the policy
         Request moveToStepRequest = new Request("POST", "_ilm/move/test-1");
@@ -188,7 +188,12 @@ public void testMoveToStepRereadsPolicy() throws Exception {
             "    \"name\": \"check-rollover-ready\"\n" +
             "  }\n" +
             "}");
-        assertOK(client().performRequest(moveToStepRequest));
+        // busy asserting here as ILM moves the index from the `check-rollover-ready` step into the `error` step and back into the
+        // `check-rollover-ready` when retrying. the `_ilm/move` api might fail when the as the `current_step` of the index might be
+        //  the `error` step at execution time.
+        assertBusy(() -> client().performRequest(moveToStepRequest), 30, TimeUnit.SECONDS);
+
+        indexDocument(client(), "test-1", true);
 
         // Make sure we actually rolled over
         assertBusy(() -> {