From 30112ba02b67c4d5f6b8af71bd3776bf4f42ca49 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 15 Dec 2020 12:05:03 +0100 Subject: [PATCH] Fix IndexFoldersDeletionListenerIT --- .../IndexFoldersDeletionListenerIT.java | 106 ++++++++++++------ 1 file changed, 73 insertions(+), 33 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java index 2829bd17a0c1d..b0df5c4fc1808 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.env.Environment; @@ -45,6 +46,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.elasticsearch.env.NodeEnvironment.INDICES_FOLDER; @@ -55,7 +57,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) public class IndexFoldersDeletionListenerIT extends ESIntegTestCase { @Override @@ -66,26 +68,37 @@ protected Collection> nodePlugins() { } public void testListenersInvokedWhenIndexIsDeleted() throws Exception { + final String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNodes(2); + ensureStableCluster(2 + 1, masterNode); + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - createIndex(indexName); + createIndex(indexName, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2 * between(1, 2)) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1)) + .build()); final NumShards numShards = getNumShards(indexName); - ensureClusterSizeConsistency(); // wait for a stable cluster - ensureGreen(indexName); // wait for no relocation - - final ClusterState clusterState = clusterService().state(); + assertFalse(client().admin().cluster().prepareHealth() + .setIndices(indexName) + .setWaitForGreenStatus() + .setWaitForEvents(Priority.LANGUID) + .setWaitForNoRelocatingShards(true) + .setWaitForNoInitializingShards(true) + .get() + .isTimedOut()); + + final ClusterState clusterState = internalCluster().clusterService(masterNode).state(); final Index index = clusterState.metadata().index(indexName).getIndex(); final Map> shardsByNodes = shardRoutingsByNodes(clusterState, index); assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards)); for (Map.Entry> shardsByNode : shardsByNodes.entrySet()) { - final String nodeName = shardsByNode.getKey(); - final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName); - assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty()); - assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty()); + assertNoDeletions(shardsByNode.getKey()); } assertAcked(client().admin().indices().prepareDelete(indexName)); + assertPendingDeletesProcessed(); assertBusy(() -> { for (Map.Entry> shardsByNode : shardsByNodes.entrySet()) { @@ -105,30 +118,37 @@ public void testListenersInvokedWhenIndexIsDeleted() throws Exception { deletedShards.contains(shardId)); } } - }); + }, 30L, TimeUnit.SECONDS); } public void testListenersInvokedWhenIndexIsRelocated() throws Exception { - internalCluster().ensureAtLeastNumDataNodes(4); + final String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNodes(4); + ensureStableCluster(4 + 1, masterNode); + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createIndex(indexName, Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10)) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2)) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1)) .build()); final NumShards numShards = getNumShards(indexName); - ensureGreen(indexName); - - final ClusterState clusterState = clusterService().state(); + assertFalse(client().admin().cluster().prepareHealth() + .setIndices(indexName) + .setWaitForGreenStatus() + .setWaitForEvents(Priority.LANGUID) + .setWaitForNoRelocatingShards(true) + .setWaitForNoInitializingShards(true) + .get() + .isTimedOut()); + + final ClusterState clusterState = internalCluster().clusterService(masterNode).state(); final Index index = clusterState.metadata().index(indexName).getIndex(); final Map> shardsByNodes = shardRoutingsByNodes(clusterState, index); assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards)); for (Map.Entry> shardsByNode : shardsByNodes.entrySet()) { - final String nodeName = shardsByNode.getKey(); - final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName); - assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty()); - assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty()); + assertNoDeletions(shardsByNode.getKey()); } final List excludedNodes = randomSubsetOf(2, shardsByNodes.keySet()); @@ -158,48 +178,58 @@ public void testListenersInvokedWhenIndexIsRelocated() throws Exception { deletedShards.contains(shardId)); } } else { - assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty()); - assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty()); + assertNoDeletions(nodeName); } } - }); + }, 30L, TimeUnit.SECONDS); } public void testListenersInvokedWhenIndexIsDangling() throws Exception { - internalCluster().ensureAtLeastNumDataNodes(4); + final String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNodes(4); + ensureStableCluster(4 + 1, masterNode); + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createIndex(indexName, Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10)) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2)) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1)) .build()); final NumShards numShards = getNumShards(indexName); - ensureGreen(indexName); - - final ClusterState clusterState = clusterService().state(); + assertFalse(client().admin().cluster().prepareHealth() + .setIndices(indexName) + .setWaitForGreenStatus() + .setWaitForEvents(Priority.LANGUID) + .setWaitForNoRelocatingShards(true) + .setWaitForNoInitializingShards(true) + .get() + .isTimedOut()); + + final ClusterState clusterState = internalCluster().clusterService(masterNode).state(); final Index index = clusterState.metadata().index(indexName).getIndex(); final Map> shardsByNodes = shardRoutingsByNodes(clusterState, index); assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards)); for (Map.Entry> shardsByNode : shardsByNodes.entrySet()) { - final String nodeName = shardsByNode.getKey(); - final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName); - assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty()); - assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty()); + assertNoDeletions(shardsByNode.getKey()); } final String stoppedNode = randomFrom(shardsByNodes.keySet()); final Settings stoppedNodeDataPathSettings = internalCluster().dataPathSettings(stoppedNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(stoppedNode)); + ensureStableCluster(3 + 1, masterNode); assertAcked(client().admin().indices().prepareDelete(indexName)); final String restartedNode = internalCluster().startNode(stoppedNodeDataPathSettings); + ensureStableCluster(4 + 1, masterNode); + assertPendingDeletesProcessed(); + assertBusy(() -> { final IndexFoldersDeletionListenerPlugin plugin = plugin(restartedNode); assertTrue("Listener should have been notified of deletion of index " + index + " on node " + restartedNode, plugin.deletedIndices.contains(index)); - }); + }, 30L, TimeUnit.SECONDS); } public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception { @@ -228,6 +258,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception { logger.debug("--> stopping data node [{}], the data left on disk will be injected as left-overs in a newer data node", dataNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); + ensureStableCluster(1, masterNode); logger.debug("--> deleting leftover indices"); assertAcked(client().admin().indices().prepareDelete("index-*")); @@ -264,6 +295,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception { dataPaths.stream().map(p -> p.toAbsolutePath().toString()).collect(Collectors.toList())) .putNull(Environment.PATH_SHARED_DATA_SETTING.getKey()) .build()); + ensureStableCluster(1 + 1, masterNode); final IndexFoldersDeletionListenerPlugin plugin = plugin(dataNode); assertTrue("Expecting no shards deleted on node " + dataNode, plugin.deletedShards.isEmpty()); @@ -328,4 +360,12 @@ private static void assertPendingDeletesProcessed() throws Exception { services.forEach(indicesService -> assertFalse(indicesService.hasUncompletedPendingDeletes())); }); } + + private static void assertNoDeletions(String nodeName) { + final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName); + assertTrue("Expecting no indices deleted on node [" + nodeName + "] but got: " + plugin.deletedIndices, + plugin.deletedIndices.isEmpty()); + assertTrue("Expecting no shards deleted on node [" + nodeName + "] but got: " + plugin.deletedShards, + plugin.deletedShards.isEmpty()); + } }