Test: wait for the cluster to recover in ClusterServiceTests before w…

…aiting for update state task results

On CI machines node recovery sometimes takes up to 2 seconds. When it happens an update cluster state task gets stuck behind the recovery and tests fail with 1 second timeout. This commit makes sure that we wait for recovery to complete before starting the clock.

src/test/java/org/elasticsearch/cluster/ClusterServiceTests.java

@@ -176,6 +176,7 @@ public void onFailure(String source, Throwable t) {
             }
         });
 
+        ensureGreen();
         assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
 
         assertThat(allNodesAcked.get(), equalTo(true));
@@ -247,6 +248,7 @@ public void onFailure(String source, Throwable t) {
             }
         });
 
+        ensureGreen();
         assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
 
         assertThat(allNodesAcked.get(), equalTo(true));
@@ -373,6 +375,7 @@ public void onFailure(String source, Throwable t) {
             }
         });
 
+        ensureGreen();
         assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
 
         assertThat(allNodesAcked.get(), equalTo(true));
@@ -447,6 +450,7 @@ public void onFailure(String source, Throwable t) {
             }
         });
 
+        ensureGreen();
         assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
 
         assertThat(allNodesAcked.get(), equalTo(false));