Re-enable FlushIT tests

These tests failed due to in flight operations on the primary shard.
Sadly, we don't have any clue on those ops. This commit unmutes
these tests and logs the acquirers when checking for ongoing ops.

1> [2018-05-02T23:10:32,145][INFO ][o.e.i.f.FlushIT          ] Third
seal: Total shards: [2], failed: [true], reason: [[1] ongoing operations
on primary], detail: []

Relates #29392

server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.indices.flush;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -501,8 +502,18 @@ private InFlightOpsResponse performInFlightOps(InFlightOpsRequest request) {
         if (indexShard.routingEntry().primary() == false) {
             throw new IllegalStateException("[" + request.shardId() +"] expected a primary shard");
         }
+        if (Assertions.ENABLED) {
+            if (logger.isTraceEnabled()) {
+                logger.trace("in flight operations {}, acquirers {}", indexShard.getActiveOperationsCount(), indexShard.getActiveOperations());
+            }
+        }
         int opCount = indexShard.getActiveOperationsCount();
-        logger.trace("{} in flight operations sampled at [{}]", request.shardId(), opCount);
+        // Need to snapshot the debug info twice as it's updated concurrently with the permit count.
+        if (Assertions.ENABLED) {
+            if (logger.isTraceEnabled()) {
+                logger.trace("in flight operations {}, acquirers {}", indexShard.getActiveOperationsCount(), indexShard.getActiveOperations());
+            }
+        }
         return new InFlightOpsResponse(opCount);
     }
 

server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java

@@ -254,8 +254,7 @@ private String syncedFlushDescription(ShardsSyncedFlushResult result) {
             result.totalShards(), result.failed(), result.failureReason(), detail);
     }
 
-    @TestLogging("_root:DEBUG")
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29392")
+    @TestLogging("_root:DEBUG,org.elasticsearch.indices.flush:TRACE")
     public void testSyncedFlushSkipOutOfSyncReplicas() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(between(2, 3));
         final int numberOfReplicas = internalCluster().numDataNodes() - 1;
@@ -297,8 +296,7 @@ public void testSyncedFlushSkipOutOfSyncReplicas() throws Exception {
         assertThat(fullResult.successfulShards(), equalTo(numberOfReplicas + 1));
     }
 
-    @TestLogging("_root:DEBUG")
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29392")
+    @TestLogging("_root:DEBUG,org.elasticsearch.indices.flush:TRACE")
     public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(between(2, 3));
         final int numberOfReplicas = internalCluster().numDataNodes() - 1;