Skip to content

Commit

Permalink
Fix IndexFoldersDeletionListenerIT (#66329)
Browse files Browse the repository at this point in the history 
The tests testListenersInvokedWhenIndexIsRelocated and 
testListenersInvokedWhenIndexIsDangling fail on CI with 
errors like:

java.lang.AssertionError: Expecting no shards deleted on node node_t4
	at __randomizedtesting.SeedInfo.seed(...)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.elasticsearch.plugins.IndexFoldersDeletionListenerIT.
testListenersInvokedWhenIndexIsDangling
(IndexFoldersDeletionListenerIT.java:189)

This commit tries to make those tests more stable by 
starting all nodes upfront, by waiting for no rebalancing/
relocations before checking the previous assertions 
and by extending a bit the time of some assertBusy() 
as on CI it can take more than 10 sec for pending 
deletes to be processed.
  • Loading branch information
@tlrx
tlrx authored Dec 17, 2020
1 parent 0a1cec2 commit bf96bca
Showing 1 changed file with 73 additions and 33 deletions.
106 changes: 73 additions & 33 deletions ...rc/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.env.Environment;
Expand All @@ -45,6 +46,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static org.elasticsearch.env.NodeEnvironment.INDICES_FOLDER;
Expand All @@ -55,7 +57,7 @@
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;

@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
public class IndexFoldersDeletionListenerIT extends ESIntegTestCase {

@Override
Expand All @@ -66,26 +68,37 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
}

public void testListenersInvokedWhenIndexIsDeleted() throws Exception {
final String masterNode = internalCluster().startMasterOnlyNode();
internalCluster().startDataOnlyNodes(2);
ensureStableCluster(2 + 1, masterNode);

final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
createIndex(indexName);
createIndex(indexName, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2 * between(1, 2))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
.build());

final NumShards numShards = getNumShards(indexName);
ensureClusterSizeConsistency(); // wait for a stable cluster
ensureGreen(indexName); // wait for no relocation

final ClusterState clusterState = clusterService().state();
assertFalse(client().admin().cluster().prepareHealth()
.setIndices(indexName)
.setWaitForGreenStatus()
.setWaitForEvents(Priority.LANGUID)
.setWaitForNoRelocatingShards(true)
.setWaitForNoInitializingShards(true)
.get()
.isTimedOut());

final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
final Index index = clusterState.metadata().index(indexName).getIndex();
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));

for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
final String nodeName = shardsByNode.getKey();
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
assertNoDeletions(shardsByNode.getKey());
}

assertAcked(client().admin().indices().prepareDelete(indexName));
assertPendingDeletesProcessed();

assertBusy(() -> {
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
Expand All @@ -105,30 +118,37 @@ public void testListenersInvokedWhenIndexIsDeleted() throws Exception {
deletedShards.contains(shardId));
}
}
});
}, 30L, TimeUnit.SECONDS);
}

public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(4);
final String masterNode = internalCluster().startMasterOnlyNode();
internalCluster().startDataOnlyNodes(4);
ensureStableCluster(4 + 1, masterNode);

final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
createIndex(indexName, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10))
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
.build());

final NumShards numShards = getNumShards(indexName);
ensureGreen(indexName);

final ClusterState clusterState = clusterService().state();
assertFalse(client().admin().cluster().prepareHealth()
.setIndices(indexName)
.setWaitForGreenStatus()
.setWaitForEvents(Priority.LANGUID)
.setWaitForNoRelocatingShards(true)
.setWaitForNoInitializingShards(true)
.get()
.isTimedOut());

final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
final Index index = clusterState.metadata().index(indexName).getIndex();
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));

for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
final String nodeName = shardsByNode.getKey();
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
assertNoDeletions(shardsByNode.getKey());
}

final List<String> excludedNodes = randomSubsetOf(2, shardsByNodes.keySet());
Expand Down Expand Up @@ -158,48 +178,58 @@ public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
deletedShards.contains(shardId));
}
} else {
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
assertNoDeletions(nodeName);
}
}
});
}, 30L, TimeUnit.SECONDS);
}

public void testListenersInvokedWhenIndexIsDangling() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(4);
final String masterNode = internalCluster().startMasterOnlyNode();
internalCluster().startDataOnlyNodes(4);
ensureStableCluster(4 + 1, masterNode);

final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
createIndex(indexName, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10))
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
.build());

final NumShards numShards = getNumShards(indexName);
ensureGreen(indexName);

final ClusterState clusterState = clusterService().state();
assertFalse(client().admin().cluster().prepareHealth()
.setIndices(indexName)
.setWaitForGreenStatus()
.setWaitForEvents(Priority.LANGUID)
.setWaitForNoRelocatingShards(true)
.setWaitForNoInitializingShards(true)
.get()
.isTimedOut());

final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
final Index index = clusterState.metadata().index(indexName).getIndex();
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));

for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
final String nodeName = shardsByNode.getKey();
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
assertNoDeletions(shardsByNode.getKey());
}

final String stoppedNode = randomFrom(shardsByNodes.keySet());
final Settings stoppedNodeDataPathSettings = internalCluster().dataPathSettings(stoppedNode);
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(stoppedNode));
ensureStableCluster(3 + 1, masterNode);

assertAcked(client().admin().indices().prepareDelete(indexName));

final String restartedNode = internalCluster().startNode(stoppedNodeDataPathSettings);
ensureStableCluster(4 + 1, masterNode);
assertPendingDeletesProcessed();

assertBusy(() -> {
final IndexFoldersDeletionListenerPlugin plugin = plugin(restartedNode);
assertTrue("Listener should have been notified of deletion of index " + index + " on node " + restartedNode,
plugin.deletedIndices.contains(index));
});
}, 30L, TimeUnit.SECONDS);
}

public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
Expand Down Expand Up @@ -228,6 +258,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {

logger.debug("--> stopping data node [{}], the data left on disk will be injected as left-overs in a newer data node", dataNode);
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode));
ensureStableCluster(1, masterNode);

logger.debug("--> deleting leftover indices");
assertAcked(client().admin().indices().prepareDelete("index-*"));
Expand Down Expand Up @@ -264,6 +295,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
dataPaths.stream().map(p -> p.toAbsolutePath().toString()).collect(Collectors.toList()))
.putNull(Environment.PATH_SHARED_DATA_SETTING.getKey())
.build());
ensureStableCluster(1 + 1, masterNode);

final IndexFoldersDeletionListenerPlugin plugin = plugin(dataNode);
assertTrue("Expecting no shards deleted on node " + dataNode, plugin.deletedShards.isEmpty());
Expand Down Expand Up @@ -328,4 +360,12 @@ private static void assertPendingDeletesProcessed() throws Exception {
services.forEach(indicesService -> assertFalse(indicesService.hasUncompletedPendingDeletes()));
});
}

private static void assertNoDeletions(String nodeName) {
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
assertTrue("Expecting no indices deleted on node [" + nodeName + "] but got: " + plugin.deletedIndices,
plugin.deletedIndices.isEmpty());
assertTrue("Expecting no shards deleted on node [" + nodeName + "] but got: " + plugin.deletedShards,
plugin.deletedShards.isEmpty());
}
}

0 comments on commit bf96bca

Please sign in to comment.