Increase waiting time when check retention locks (#42994)

WriteActionsTests#testBulk and WriteActionsTests#testIndex sometimes
fail with a pending retention lock. We might leak retention locks when
switching to async recovery. However, it's more likely that ongoing
recoveries prevent the retention lock from releasing.

This change increases the waiting time when we check for no pending
retention lock and also ensures no ongoing recovery in
WriteActionsTests.

Closes #41054

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -1351,7 +1351,7 @@ private void assertOpenTranslogReferences() throws Exception {
                     }
                 }
             }
-        });
+        }, 60, TimeUnit.SECONDS);
     }
 
     private void assertNoSnapshottedIndexCommit() throws Exception {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java

@@ -53,6 +53,7 @@ public void testIndex() {
         //the missing index gets automatically created (user has permissions for that), but indexing fails due to missing authorization
         assertThrowsAuthorizationExceptionDefaultUsers(client().prepareIndex("missing", "type", "id").setSource("field", "value")::get,
                 BulkAction.NAME + "[s]");
+        ensureGreen();
     }
 
     public void testDelete() {
@@ -63,6 +64,7 @@ public void testDelete() {
         assertThrowsAuthorizationExceptionDefaultUsers(client().prepareDelete("index1", "type", "id")::get, BulkAction.NAME + "[s]");
 
         expectThrows(IndexNotFoundException.class, () -> client().prepareDelete("test4", "type", "id").get());
+        ensureGreen();
     }
 
     public void testUpdate() {
@@ -79,6 +81,7 @@ public void testUpdate() {
 
         assertThrowsAuthorizationExceptionDefaultUsers(client().prepareUpdate("missing", "type", "id")
                         .setDoc(Requests.INDEX_CONTENT_TYPE, "field2", "value2")::get, UpdateAction.NAME);
+        ensureGreen();
     }
 
     public void testBulk() {
@@ -160,5 +163,6 @@ public void testBulk() {
         assertAuthorizationExceptionDefaultUsers(bulkResponse.getItems()[12].getFailure().getCause(), BulkAction.NAME + "[s]");
         assertThat(bulkResponse.getItems()[12].getFailure().getCause().getMessage(),
                 containsString("[indices:data/write/bulk[s]] is unauthorized"));
+        ensureGreen();
     }
 }