From 309a3e4ccbcdfcd830621a92d6e67af929123605 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 28 Feb 2019 18:35:33 +0100 Subject: [PATCH] Add support for replicating closed indices (#39499) Before this change, closed indexes were simply not replicated. It was therefore possible to close an index and then decommission a data node without knowing that this data node contained shards of the closed index, potentially leading to data loss. Shards of closed indices were not completely taken into account when balancing the shards within the cluster, or automatically replicated through shard copies, and they were not easily movable from node A to node B using APIs like Cluster Reroute without being fully reopened and closed again. This commit changes the logic executed when closing an index, so that its shards are not just removed and forgotten but are instead reinitialized and reallocated on data nodes using an engine implementation which does not allow searching or indexing, which has a low memory overhead (compared with searchable/indexable opened shards) and which allows shards to be recovered from peer or promoted as primaries when needed. This new closing logic is built on top of the new Close Index API introduced in 6.7.0 (#37359). Some pre-closing sanity checks are executed on the shards before closing them, and closing an index on a 8.0 cluster will reinitialize the index shards and therefore impact the cluster health. Some APIs have been adapted to make them work with closed indices: - Cluster Health API - Cluster Reroute API - Cluster Allocation Explain API - Recovery API - Cat Indices - Cat Shards - Cat Health - Cat Recovery This commit contains all the following changes (most recent first): * c6c42a1 Adapt NoOpEngineTests after #39006 * 3f9993d Wait for shards to be active after closing indices (#38854) * 5e7a428 Adapt the Cluster Health API to closed indices (#39364) * 3e61939 Adapt CloseFollowerIndexIT for replicated closed indices (#38767) * 71f5c34 Recover closed indices after a full cluster restart (#39249) * 4db7fd9 Adapt the Recovery API for closed indices (#38421) * 4fd1bb2 Adapt more tests suites to closed indices (#39186) * 0519016 Add replica to primary promotion test for closed indices (#39110) * b756f6c Test the Cluster Shard Allocation Explain API with closed indices (#38631) * c484c66 Remove index routing table of closed indices in mixed versions clusters (#38955) * 00f1828 Mute CloseFollowerIndexIT.testCloseAndReopenFollowerIndex() * e845b0a Do not schedule Refresh/Translog/GlobalCheckpoint tasks for closed indices (#38329) * cf9a015 Adapt testIndexCanChangeCustomDataPath for replicated closed indices (#38327) * b9becdd Adapt testPendingTasks() for replicated closed indices (#38326) * 02cc730 Allow shards of closed indices to be replicated as regular shards (#38024) * e53a9be Fix compilation error in IndexShardIT after merge with master * cae4155 Relax NoOpEngine constraints (#37413) * 54d110b [RCI] Adapt NoOpEngine to latest FrozenEngine changes * c63fd69 [RCI] Add NoOpEngine for closed indices (#33903) Relates to #33888 --- .../upgrades/FullClusterRestartIT.java | 96 +++++++++++ .../elasticsearch/upgrades/RecoveryIT.java | 147 ++++++++++++++++ .../rest-api-spec/api/cluster.health.json | 6 + .../rest-api-spec/api/indices.close.json | 4 + .../test/cat.indices/10_basic.yml | 85 ++++++++-- .../test/cat.recovery/10_basic.yml | 57 +++++++ .../cluster.allocation_explain/10_basic.yml | 44 +++++ .../test/cluster.health/10_basic.yml | 146 ++++++++++++++++ .../cluster.health/30_indices_options.yml | 79 +++++++++ .../test/indices.open/10_basic.yml | 24 +++ .../test/indices.open/20_multiple_indices.yml | 6 + .../test/indices.recovery/10_basic.yml | 50 ++++++ .../cluster/health/ClusterHealthRequest.java | 16 +- .../health/ClusterHealthRequestBuilder.java | 6 + .../admin/indices/close/CloseIndexAction.java | 7 +- .../CloseIndexClusterStateUpdateRequest.java | 18 +- .../indices/close/CloseIndexRequest.java | 20 +++ .../close/CloseIndexRequestBuilder.java | 31 +++- .../indices/close/CloseIndexResponse.java | 52 ++++++ .../close/TransportCloseIndexAction.java | 27 +-- .../indices/recovery/RecoveryRequest.java | 3 +- .../recovery/TransportRecoveryAction.java | 4 +- .../action/support/IndicesOptions.java | 14 +- .../support/broadcast/BroadcastRequest.java | 5 + .../client/IndicesAdminClient.java | 5 +- .../client/support/AbstractClient.java | 5 +- .../metadata/MetaDataIndexStateService.java | 92 +++++++--- .../cluster/routing/IndexRoutingTable.java | 7 + .../cluster/routing/RoutingTable.java | 13 +- .../cluster/routing/UnassignedInfo.java | 8 +- .../common/settings/IndexScopedSettings.java | 2 + .../org/elasticsearch/index/IndexService.java | 11 +- .../index/engine/NoOpEngine.java | 89 ++++++++++ .../elasticsearch/indices/IndicesService.java | 7 + .../cluster/IndicesClusterStateService.java | 47 ++++-- .../cluster/RestClusterHealthAction.java | 6 +- .../admin/indices/RestCloseIndexAction.java | 5 + .../rest/action/cat/RestIndicesAction.java | 60 ++++--- .../elasticsearch/search/SearchService.java | 2 +- .../ClusterAllocationExplainIT.java | 83 ++++++--- .../health/ClusterHealthRequestTests.java | 104 +++++++++++- .../indices/close/CloseIndexRequestTests.java | 114 +++++++++++++ .../close/CloseIndexResponseTests.java | 86 ++++++++++ .../cluster/ClusterHealthIT.java | 159 ++++++++++++++++++ .../allocation/AwarenessAllocationIT.java | 109 +++++++++--- .../cluster/allocation/ClusterRerouteIT.java | 93 +++++++--- .../allocation/FilteringAllocationIT.java | 57 ++++--- .../allocation/SimpleAllocationIT.java | 18 +- .../IndexNameExpressionResolverTests.java | 7 +- .../MetaDataIndexStateServiceTests.java | 152 +++++++++++++---- .../cluster/routing/RoutingTableTests.java | 32 ++++ .../cluster/routing/UnassignedInfoTests.java | 20 ++- .../gateway/ClusterStateUpdatersTests.java | 36 +++- .../gateway/GatewayIndexStateIT.java | 19 ++- .../index/IndexServiceTests.java | 80 ++++++++- .../index/engine/NoOpEngineRecoveryTests.java | 54 ++++++ .../index/engine/NoOpEngineTests.java | 158 +++++++++++++++++ .../index/shard/IndexShardIT.java | 122 ++++++++------ .../indices/IndicesLifecycleListenerIT.java | 10 +- .../indices/IndicesServiceTests.java | 78 +++++---- .../indices/recovery/IndexRecoveryIT.java | 56 ++++-- .../recovery/ReplicaToPrimaryPromotionIT.java | 87 ++++++++++ .../indices/state/CloseIndexIT.java | 39 ++++- .../indices/state/ReopenWhileClosingIT.java | 6 +- .../indices/state/SimpleIndexStateIT.java | 6 +- .../search/scroll/SearchScrollIT.java | 11 +- .../SharedClusterSnapshotRestoreIT.java | 4 +- .../index/engine/EngineTestCase.java | 8 + x-pack/plugin/ccr/build.gradle | 14 ++ .../ccr/action/ShardFollowTasksExecutor.java | 3 +- .../xpack/ccr/CloseFollowerIndexIT.java | 60 ++++++- .../xpack/ccr/IndexFollowingIT.java | 2 +- .../action/TransportFreezeIndexAction.java | 5 +- .../index/engine/FrozenIndexTests.java | 2 +- .../CloseFollowerIndexStepTests.java | 6 +- 75 files changed, 2790 insertions(+), 416 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/30_indices_options.yml create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java create mode 100644 server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/engine/NoOpEngineRecoveryTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java create mode 100644 server/src/test/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index b34f677e1c15b..36cb6bcadd73b 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; @@ -41,6 +42,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -59,8 +61,11 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * Tests to run before and after a full cluster restart. This is run twice, @@ -951,6 +956,97 @@ public void testSoftDeletes() throws Exception { } } + /** + * This test creates an index in the old cluster and then closes it. When the cluster is fully restarted in a newer version, + * it verifies that the index exists and is replicated if the old version supports replication. + */ + public void testClosedIndices() throws Exception { + if (isRunningAgainstOldCluster()) { + createIndex(index, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .build()); + ensureGreen(index); + + int numDocs = 0; + if (randomBoolean()) { + numDocs = between(1, 100); + for (int i = 0; i < numDocs; i++) { + final Request request = new Request("POST", "/" + index + "/_doc/" + i); + request.setJsonEntity(Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject())); + assertOK(client().performRequest(request)); + if (rarely()) { + refresh(); + } + } + refresh(); + } + + assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")))); + saveInfoDocument(index + "_doc_count", Integer.toString(numDocs)); + closeIndex(index); + } + + if (getOldClusterVersion().onOrAfter(Version.V_8_0_0)) { + ensureGreenLongWait(index); + assertClosedIndex(index, true); + } else { + assertClosedIndex(index, false); + } + + if (isRunningAgainstOldCluster() == false) { + openIndex(index); + ensureGreen(index); + + final int expectedNumDocs = Integer.parseInt(loadInfoDocument(index + "_doc_count")); + assertTotalHits(expectedNumDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")))); + } + } + + /** + * Asserts that an index is closed in the cluster state. If `checkRoutingTable` is true, it also asserts + * that the index has started shards. + */ + @SuppressWarnings("unchecked") + private void assertClosedIndex(final String index, final boolean checkRoutingTable) throws IOException { + final Map state = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state"))); + + final Map metadata = (Map) XContentMapValues.extractValue("metadata.indices." + index, state); + assertThat(metadata, notNullValue()); + assertThat(metadata.get("state"), equalTo("close")); + + final Map blocks = (Map) XContentMapValues.extractValue("blocks.indices." + index, state); + assertThat(blocks, notNullValue()); + assertThat(blocks.containsKey(String.valueOf(MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID)), is(true)); + + final Map settings = (Map) XContentMapValues.extractValue("settings", metadata); + assertThat(settings, notNullValue()); + + final Map routingTable = (Map) XContentMapValues.extractValue("routing_table.indices." + index, state); + if (checkRoutingTable) { + assertThat(routingTable, notNullValue()); + assertThat(Booleans.parseBoolean((String) XContentMapValues.extractValue("index.verified_before_close", settings)), is(true)); + final String numberOfShards = (String) XContentMapValues.extractValue("index.number_of_shards", settings); + assertThat(numberOfShards, notNullValue()); + final int nbShards = Integer.parseInt(numberOfShards); + assertThat(nbShards, greaterThanOrEqualTo(1)); + + for (int i = 0; i < nbShards; i++) { + final Collection> shards = + (Collection>) XContentMapValues.extractValue("shards." + i, routingTable); + assertThat(shards, notNullValue()); + assertThat(shards.size(), equalTo(2)); + for (Map shard : shards) { + assertThat(XContentMapValues.extractValue("shard", shard), equalTo(i)); + assertThat(XContentMapValues.extractValue("state", shard), equalTo("STARTED")); + assertThat(XContentMapValues.extractValue("index", shard), equalTo(index)); + } + } + } else { + assertThat(routingTable, nullValue()); + assertThat(XContentMapValues.extractValue("index.verified_before_close", settings), nullValue()); + } + } + private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException { // Check the snapshot metadata, especially the version Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName); diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index 295aee8b869ff..402c0c4859b77 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -24,15 +24,20 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.test.rest.yaml.ObjectPath; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.Future; import java.util.function.Predicate; @@ -43,7 +48,9 @@ import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** * In depth testing of the recovery mechanism during a rolling restart. @@ -310,4 +317,144 @@ public void testRecoveryWithSoftDeletes() throws Exception { } ensureGreen(index); } + + /** + * This test creates an index in the non upgraded cluster and closes it. It then checks that the index + * is effectively closed and potentially replicated (if the version the index was created on supports + * the replication of closed indices) during the rolling upgrade. + */ + public void testRecoveryClosedIndex() throws Exception { + final String indexName = "closed_index_created_on_old"; + if (CLUSTER_TYPE == ClusterType.OLD) { + createIndex(indexName, Settings.builder() + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1) + // if the node with the replica is the first to be restarted, while a replica is still recovering + // then delayed allocation will kick in. When the node comes back, the master will search for a copy + // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN + // before timing out + .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster + .build()); + ensureGreen(indexName); + closeIndex(indexName); + } + + final Version indexVersionCreated = indexVersionCreated(indexName); + if (indexVersionCreated.onOrAfter(Version.V_8_0_0)) { + // index was created on a version that supports the replication of closed indices, + // so we expect the index to be closed and replicated + ensureGreen(indexName); + assertClosedIndex(indexName, true); + } else { + assertClosedIndex(indexName, false); + } + } + + /** + * This test creates and closes a new index at every stage of the rolling upgrade. It then checks that the index + * is effectively closed and potentially replicated if the cluster supports replication of closed indices at the + * time the index was closed. + */ + public void testCloseIndexDuringRollingUpgrade() throws Exception { + final Version minimumNodeVersion = minimumNodeVersion(); + final String indexName = + String.join("_", "index", CLUSTER_TYPE.toString(), Integer.toString(minimumNodeVersion.id)).toLowerCase(Locale.ROOT); + + if (indexExists(indexName) == false) { + createIndex(indexName, Settings.builder() + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) + .build()); + ensureGreen(indexName); + closeIndex(indexName); + } + + if (minimumNodeVersion.onOrAfter(Version.V_8_0_0)) { + // index is created on a version that supports the replication of closed indices, + // so we expect the index to be closed and replicated + ensureGreen(indexName); + assertClosedIndex(indexName, true); + } else { + assertClosedIndex(indexName, false); + } + } + + /** + * Returns the version in which the given index has been created + */ + private static Version indexVersionCreated(final String indexName) throws IOException { + final Request request = new Request("GET", "/" + indexName + "/_settings"); + final String versionCreatedSetting = indexName + ".settings.index.version.created"; + request.addParameter("filter_path", versionCreatedSetting); + + final Response response = client().performRequest(request); + return Version.fromId(Integer.parseInt(ObjectPath.createFromResponse(response).evaluate(versionCreatedSetting))); + } + + /** + * Returns the minimum node version among all nodes of the cluster + */ + private static Version minimumNodeVersion() throws IOException { + final Request request = new Request("GET", "_nodes"); + request.addParameter("filter_path", "nodes.*.version"); + + final Response response = client().performRequest(request); + final Map nodes = ObjectPath.createFromResponse(response).evaluate("nodes"); + + Version minVersion = null; + for (Map.Entry node : nodes.entrySet()) { + @SuppressWarnings("unchecked") + Version nodeVersion = Version.fromString((String) ((Map) node.getValue()).get("version")); + if (minVersion == null || minVersion.after(nodeVersion)) { + minVersion = nodeVersion; + } + } + assertNotNull(minVersion); + return minVersion; + } + + /** + * Asserts that an index is closed in the cluster state. If `checkRoutingTable` is true, it also asserts + * that the index has started shards. + */ + @SuppressWarnings("unchecked") + private void assertClosedIndex(final String index, final boolean checkRoutingTable) throws IOException { + final Map state = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state"))); + + final Map metadata = (Map) XContentMapValues.extractValue("metadata.indices." + index, state); + assertThat(metadata, notNullValue()); + assertThat(metadata.get("state"), equalTo("close")); + + final Map blocks = (Map) XContentMapValues.extractValue("blocks.indices." + index, state); + assertThat(blocks, notNullValue()); + assertThat(blocks.containsKey(String.valueOf(MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID)), is(true)); + + final Map settings = (Map) XContentMapValues.extractValue("settings", metadata); + assertThat(settings, notNullValue()); + + final int numberOfShards = Integer.parseInt((String) XContentMapValues.extractValue("index.number_of_shards", settings)); + final int numberOfReplicas = Integer.parseInt((String) XContentMapValues.extractValue("index.number_of_replicas", settings)); + + final Map routingTable = (Map) XContentMapValues.extractValue("routing_table.indices." + index, state); + if (checkRoutingTable) { + assertThat(routingTable, notNullValue()); + assertThat(Booleans.parseBoolean((String) XContentMapValues.extractValue("index.verified_before_close", settings)), is(true)); + + for (int i = 0; i < numberOfShards; i++) { + final Collection> shards = + (Collection>) XContentMapValues.extractValue("shards." + i, routingTable); + assertThat(shards, notNullValue()); + assertThat(shards.size(), equalTo(numberOfReplicas + 1)); + for (Map shard : shards) { + assertThat(XContentMapValues.extractValue("shard", shard), equalTo(i)); + assertThat(XContentMapValues.extractValue("state", shard), equalTo("STARTED")); + assertThat(XContentMapValues.extractValue("index", shard), equalTo(index)); + } + } + } else { + assertThat(routingTable, nullValue()); + assertThat(XContentMapValues.extractValue("index.verified_before_close", settings), nullValue()); + } + } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json index ee32a87c927ed..49d0ef5c83aee 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json @@ -12,6 +12,12 @@ } }, "params": { + "expand_wildcards": { + "type" : "enum", + "options" : ["open","closed","none","all"], + "default" : "all", + "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, "level": { "type" : "enum", "options" : ["cluster","indices","shards"], diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json index 4eaa93030ee7b..55fd245f26c91 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json @@ -34,6 +34,10 @@ "options" : ["open","closed","none","all"], "default" : "open", "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "wait_for_active_shards": { + "type" : "string", + "description" : "Sets the number of active shards to wait for before the operation returns." } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yml index c7eddf42d1b03..9fb95c70c0b10 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yml @@ -1,5 +1,5 @@ --- -"Test cat indices output": +"Test cat indices output (no indices)": - do: cat.indices: {} @@ -7,6 +7,8 @@ - match: $body: | /^$/ +--- +"Test cat indices output": - do: indices.create: @@ -47,29 +49,88 @@ (\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d.\d\d\dZ) \s* ) $/ +--- +"Test cat indices output for closed index (pre 8.0.0)": + - skip: + version: "8.0.0 - " + reason: "closed indices are replicated starting version 8.0" + + - do: + indices.create: + index: index-2 + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + - do: indices.close: - index: index1 + index: index-2 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: green - do: cat.indices: - index: index* + index: index-* - match: $body: | - /^( \s+ - close \s+ - index1 \s+ + /^( \s+ + close \s+ + index-2 \s+ ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+ - \s+ - \s+ - \s+ - \s+ - \s+ - \s* + \s+ + \s+ + \s+ + \s+ + \s+ + \s* ) $/ +--- +"Test cat indices output for closed index": + - skip: + version: " - 7.99.99" + reason: "closed indices are replicated starting version 8.0" + - do: + indices.create: + index: index-2 + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + + - do: + indices.close: + index: index-2 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: green + + - do: + cat.indices: + index: index-* + + - match: + $body: | + /^(green \s+ + close \s+ + index-2 \s+ + ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+ + 3 \s+ + 0 \s+ + \s+ + \s+ + \s+ + \s* + ) + $/ --- "Test cat indices using health status": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml index 69ceccc1ef3bf..83194db1fa5a3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml @@ -75,3 +75,60 @@ \n )+ $/ + +--- +"Test cat recovery output for closed index": + - skip: + version: " - 7.99.99" + reason: closed indices are replicated starting version 8.0.0 + + - do: + indices.create: + index: index2 + body: + settings: + index: + number_of_replicas: 0 + + - do: + indices.close: + index: index2 + - is_true: acknowledged + + - do: + cluster.health: + index: index2 + wait_for_status: green + + - do: + cat.recovery: + index: index2 + h: i,s,t,ty,st,shost,thost,rep,snap,f,fr,fp,tf,b,br,bp,tb,to,tor,top + + - match: + $body: | + /^ + ( + index2 \s+ + \d \s+ # shard + (?:\d+ms|\d+(?:\.\d+)?s) \s+ # time in ms or seconds + existing_store \s+ # source type (always existing_store for closed indices) + done \s+ # stage + [-\w./]+ \s+ # source_host + [-\w./]+ \s+ # target_host + [-\w./]+ \s+ # repository + [-\w./]+ \s+ # snapshot + \d+ \s+ # files + \d+ \s+ # files_recovered + \d+\.\d+% \s+ # files_percent + \d+ \s+ # files_total + \d+ \s+ # bytes + \d+ \s+ # bytes_recovered + \d+\.\d+% \s+ # bytes_percent + \d+ \s+ # bytes_total + 0 \s+ # translog_ops (always 0 for closed indices) + 0 \s+ # translog_ops_recovered (always 0 for closed indices) + 100\.0% # translog_ops_percent (always 100.0% for closed indices) + \n + )+ + $/ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml index 7dbc57dac8b56..510a872f2b5b9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml @@ -53,3 +53,47 @@ - match: { primary: false } - is_true: cluster_info - is_true: can_allocate + + +--- +"Cluster shard allocation explanation test with a closed index": + - skip: + version: " - 7.99.99" + reason: closed indices are replicated starting version 8.0.0 + + - do: + indices.create: + index: test_closed + body: { "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 0 } } + + - match: { acknowledged: true } + + - do: + cluster.health: + index: test_closed + wait_for_status: green + + - do: + indices.close: + index: test_closed + + - match: { acknowledged: true } + + - do: + cluster.health: + index: test_closed + wait_for_status: green + + - do: + cluster.allocation_explain: + body: { "index": "test_closed", "shard": 0, "primary": true } + + - match: { current_state: "started" } + - is_true: current_node.id + - match: { index: "test_closed" } + - match: { shard: 0 } + - match: { primary: true } + - is_true: can_remain_on_current_node + - is_true: can_rebalance_cluster + - is_true: can_rebalance_to_other_node + - is_true: rebalance_explanation diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/10_basic.yml index 70d7372234a51..73e162c80533b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/10_basic.yml @@ -132,4 +132,150 @@ - is_true: indices - is_true: indices.test_index.shards +--- +"cluster health with closed index (pre 8.0)": + - skip: + version: "8.0.0 - " + reason: "closed indices are replicated starting version 8.0" + + - do: + indices.create: + index: index-1 + body: + settings: + index: + number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + - match: { status: green } + + - do: + indices.create: + index: index-2 + body: + settings: + index: + number_of_replicas: 50 + + - do: + cluster.health: + wait_for_status: yellow + wait_for_no_relocating_shards: true + - match: { status: yellow } + + - do: + cluster.health: + index: index-* + - match: { status: yellow } + + - do: + cluster.health: + index: index-1 + - match: { status: green } + + - do: + cluster.health: + index: index-2 + - match: { status: yellow } + + - do: + indices.close: + index: index-2 + - is_true: acknowledged + + # closing the index-2 turns the cluster health back to green + - do: + cluster.health: + wait_for_status: green + - match: { status: green } + + - do: + cluster.health: + index: index-* + - match: { status: green } + + - do: + cluster.health: + index: index-1 + - match: { status: green } + + - do: + cluster.health: + index: index-2 + - match: { status: green } +--- +"cluster health with closed index": + - skip: + version: " - 7.99.99" + reason: "closed indices are replicated starting version 8.0" + + - do: + indices.create: + index: index-1 + body: + settings: + index: + number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + - match: { status: green } + + - do: + indices.create: + index: index-2 + body: + settings: + index: + number_of_replicas: 50 + + - do: + cluster.health: + wait_for_status: yellow + wait_for_no_relocating_shards: true + - match: { status: yellow } + + - do: + cluster.health: + index: index-* + - match: { status: yellow } + + - do: + cluster.health: + index: index-1 + - match: { status: green } + + - do: + cluster.health: + index: index-2 + - match: { status: yellow } + + # closing the index-2 does not change the cluster health with replicated closed indices + - do: + indices.close: + index: index-2 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + - match: { status: yellow } + + - do: + cluster.health: + index: index-* + - match: { status: yellow } + + - do: + cluster.health: + index: index-1 + - match: { status: green } + + - do: + cluster.health: + index: index-2 + - match: { status: yellow } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/30_indices_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/30_indices_options.yml new file mode 100644 index 0000000000000..bb8f9d01e0007 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.health/30_indices_options.yml @@ -0,0 +1,79 @@ +setup: + + - do: + indices.create: + index: index-1 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + indices.create: + index: index-2 + body: + settings: + number_of_shards: 2 + number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + + - do: + indices.close: + index: index-2 + + - do: + cluster.health: + wait_for_status: green + +--- +"cluster health with expand_wildcards": + - skip: + version: " - 7.99.99" + reason: "indices options has been introduced in cluster health request starting version 8.0" + + - do: + cluster.health: + index: "index-*" + level: indices + expand_wildcards: open + - match: { status: green } + - match: { active_shards: 1 } + - match: { indices.index-1.status: green } + - match: { indices.index-1.active_shards: 1 } + - is_false: indices.index-2 + + - do: + cluster.health: + index: "index-*" + level: indices + expand_wildcards: closed + - match: { status: green } + - match: { active_shards: 2 } + - is_false: indices.index-1 + - match: { indices.index-2.status: green } + - match: { indices.index-2.active_shards: 2 } + + - do: + cluster.health: + index: "index-*" + level: indices + expand_wildcards: all + - match: { status: green } + - match: { active_shards: 3 } + - match: { indices.index-1.status: green } + - match: { indices.index-1.active_shards: 1 } + - match: { indices.index-2.status: green } + - match: { indices.index-2.active_shards: 2 } + + - do: + cluster.health: + index: "index-*" + level: indices + expand_wildcards: none + - match: { status: green } + - match: { active_shards: 0 } + - is_false: indices.index-1 + - is_false: indices.index-2 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml index 64e59d5939287..a389fee9bf761 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml @@ -14,6 +14,7 @@ - do: indices.close: index: test_index + - is_true: acknowledged - do: catch: bad_request @@ -24,6 +25,7 @@ - do: indices.open: index: test_index + - is_true: acknowledged - do: cluster.health: @@ -50,11 +52,33 @@ - do: indices.close: index: test_index + - is_true: acknowledged - do: indices.open: index: test_index wait_for_active_shards: all + - is_true: acknowledged + - match: { acknowledged: true } + - match: { shards_acknowledged: true } + +--- +"Close index with wait_for_active_shards set to all": + - skip: + version: " - 7.99.99" + reason: "closed indices are replicated starting version 8.0" + + - do: + indices.create: + index: test_index + body: + settings: + number_of_replicas: 0 + - do: + indices.close: + index: test_index + wait_for_active_shards: all + - is_true: acknowledged - match: { acknowledged: true } - match: { shards_acknowledged: true } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml index 8e1bf660f6378..bef5ea8a54651 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml @@ -26,6 +26,7 @@ setup: - do: indices.close: index: _all + - is_true: acknowledged - do: catch: bad_request @@ -36,6 +37,7 @@ setup: - do: indices.open: index: _all + - is_true: acknowledged - do: cluster.health: @@ -51,6 +53,7 @@ setup: - do: indices.close: index: test_* + - is_true: acknowledged - do: catch: bad_request @@ -61,6 +64,7 @@ setup: - do: indices.open: index: test_* + - is_true: acknowledged - do: cluster.health: @@ -76,6 +80,7 @@ setup: - do: indices.close: index: '*' + - is_true: acknowledged - do: catch: bad_request @@ -86,6 +91,7 @@ setup: - do: indices.open: index: '*' + - is_true: acknowledged - do: cluster.health: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml index fd8937a23cdee..07fe657e77ff3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml @@ -40,6 +40,56 @@ - gte: { test_1.shards.0.verify_index.check_index_time_in_millis: 0 } - gte: { test_1.shards.0.verify_index.total_time_in_millis: 0 } --- +"Indices recovery test for closed index": + - skip: + version: " - 7.99.99" + reason: closed indices are replicated starting version 8.0.0 + + - do: + indices.create: + index: test_2 + body: + settings: + index: + number_of_replicas: 0 + + - do: + indices.close: + index: test_2 + - is_true: acknowledged + + - do: + cluster.health: + index: test_2 + wait_for_status: green + + - do: + indices.recovery: + index: [test_2] + human: true + + - match: { test_2.shards.0.type: "EXISTING_STORE" } + - match: { test_2.shards.0.stage: "DONE" } + - match: { test_2.shards.0.primary: true } + - match: { test_2.shards.0.start_time: /^2\d\d\d-.+/ } + - match: { test_2.shards.0.target.ip: /^\d+\.\d+\.\d+\.\d+$/ } + - gte: { test_2.shards.0.index.files.total: 0 } + - gte: { test_2.shards.0.index.files.reused: 0 } + - gte: { test_2.shards.0.index.files.recovered: 0 } + - match: { test_2.shards.0.index.files.percent: /^\d+\.\d\%$/ } + - gte: { test_2.shards.0.index.size.total_in_bytes: 0 } + - gte: { test_2.shards.0.index.size.reused_in_bytes: 0 } + - gte: { test_2.shards.0.index.size.recovered_in_bytes: 0 } + - match: { test_2.shards.0.index.size.percent: /^\d+\.\d\%$/ } + - gte: { test_2.shards.0.index.source_throttle_time_in_millis: 0 } + - gte: { test_2.shards.0.index.target_throttle_time_in_millis: 0 } + - gte: { test_2.shards.0.translog.recovered: 0 } + - gte: { test_2.shards.0.translog.total: 0 } + - gte: { test_2.shards.0.translog.total_on_start: 0 } + - gte: { test_2.shards.0.translog.total_time_in_millis: 0 } + - gte: { test_2.shards.0.verify_index.check_index_time_in_millis: 0 } + - gte: { test_2.shards.0.verify_index.total_time_in_millis: 0 } +--- "Indices recovery test index name not matching": - do: diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java index 0b9bcbf11b9a1..03a8e52dd60ed 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -39,6 +39,7 @@ public class ClusterHealthRequest extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private String[] indices; + private IndicesOptions indicesOptions = IndicesOptions.lenientExpand(); private TimeValue timeout = new TimeValue(30, TimeUnit.SECONDS); private ClusterHealthStatus waitForStatus; private boolean waitForNoRelocatingShards = false; @@ -83,6 +84,11 @@ public ClusterHealthRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_6_2_0)) { waitForNoInitializingShards = in.readBoolean(); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + indicesOptions = IndicesOptions.readIndicesOptions(in); + } else { + indicesOptions = IndicesOptions.lenientExpandOpen(); + } } @Override @@ -115,6 +121,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_2_0)) { out.writeBoolean(waitForNoInitializingShards); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + indicesOptions.writeIndicesOptions(out); + } } @Override @@ -130,7 +139,12 @@ public ClusterHealthRequest indices(String... indices) { @Override public IndicesOptions indicesOptions() { - return IndicesOptions.lenientExpandOpen(); + return indicesOptions; + } + + public ClusterHealthRequest indicesOptions(final IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; } public TimeValue timeout() { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java index c79aac2afaf1a..21bf3357d5be4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.cluster.health; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -38,6 +39,11 @@ public ClusterHealthRequestBuilder setIndices(String... indices) { return this; } + public ClusterHealthRequestBuilder setIndicesOptions(final IndicesOptions indicesOptions) { + request.indicesOptions(indicesOptions); + return this; + } + public ClusterHealthRequestBuilder setTimeout(TimeValue timeout) { request.timeout(timeout); return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java index 68a911ff58627..5c3d60dd44013 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java @@ -20,9 +20,8 @@ package org.elasticsearch.action.admin.indices.close; import org.elasticsearch.action.Action; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -public class CloseIndexAction extends Action { +public class CloseIndexAction extends Action { public static final CloseIndexAction INSTANCE = new CloseIndexAction(); public static final String NAME = "indices:admin/close"; @@ -32,7 +31,7 @@ private CloseIndexAction() { } @Override - public AcknowledgedResponse newResponse() { - return new AcknowledgedResponse(); + public CloseIndexResponse newResponse() { + return new CloseIndexResponse(); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java index bb0f98ac07b7e..955ddf6fe8a76 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.action.admin.indices.close; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; /** @@ -25,7 +26,8 @@ */ public class CloseIndexClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest { - private final long taskId; + private long taskId; + private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; public CloseIndexClusterStateUpdateRequest(final long taskId) { this.taskId = taskId; @@ -34,4 +36,18 @@ public CloseIndexClusterStateUpdateRequest(final long taskId) { public long taskId() { return taskId; } + + public CloseIndexClusterStateUpdateRequest taskId(final long taskId) { + this.taskId = taskId; + return this; + } + + public ActiveShardCount waitForActiveShards() { + return waitForActiveShards; + } + + public CloseIndexClusterStateUpdateRequest waitForActiveShards(final ActiveShardCount waitForActiveShards) { + this.waitForActiveShards = waitForActiveShards; + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java index 272bae9425712..10bba4359c4a1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java @@ -19,8 +19,10 @@ package org.elasticsearch.action.admin.indices.close; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,6 +40,7 @@ public class CloseIndexRequest extends AcknowledgedRequest im private String[] indices; private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen(); + private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; // Changes this to NONE on 7.x to keep previous behavior public CloseIndexRequest() { } @@ -101,11 +104,25 @@ public CloseIndexRequest indicesOptions(IndicesOptions indicesOptions) { return this; } + public ActiveShardCount waitForActiveShards() { + return waitForActiveShards; + } + + public CloseIndexRequest waitForActiveShards(final ActiveShardCount waitForActiveShards) { + this.waitForActiveShards = waitForActiveShards; + return this; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + waitForActiveShards = ActiveShardCount.readFrom(in); + } else { + waitForActiveShards = ActiveShardCount.NONE; + } } @Override @@ -113,5 +130,8 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + waitForActiveShards.writeTo(out); + } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java index e69c6fed87dcd..7db79e0c3e550 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java @@ -19,16 +19,16 @@ package org.elasticsearch.action.admin.indices.close; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.ElasticsearchClient; /** * Builder for close index request */ public class CloseIndexRequestBuilder - extends AcknowledgedRequestBuilder { + extends AcknowledgedRequestBuilder { public CloseIndexRequestBuilder(ElasticsearchClient client, CloseIndexAction action) { super(client, action, new CloseIndexRequest()); @@ -60,4 +60,31 @@ public CloseIndexRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) request.indicesOptions(indicesOptions); return this; } + + /** + * Sets the number of shard copies that should be active for indices closing to return. + * Defaults to {@link ActiveShardCount#DEFAULT}, which will wait for one shard copy + * (the primary) to become active. Set this value to {@link ActiveShardCount#ALL} to + * wait for all shards (primary and all replicas) to be active before returning. + * Otherwise, use {@link ActiveShardCount#from(int)} to set this value to any + * non-negative integer, up to the number of copies per shard (number of replicas + 1), + * to wait for the desired amount of shard copies to become active before returning. + * Indices closing will only wait up until the timeout value for the number of shard copies + * to be active before returning. + * + * @param waitForActiveShards number of active shard copies to wait on + */ + public CloseIndexRequestBuilder setWaitForActiveShards(final ActiveShardCount waitForActiveShards) { + request.waitForActiveShards(waitForActiveShards); + return this; + } + + /** + * A shortcut for {@link #setWaitForActiveShards(ActiveShardCount)} where the numerical + * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)} + * to get the ActiveShardCount. + */ + public CloseIndexRequestBuilder setWaitForActiveShards(final int waitForActiveShards) { + return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java new file mode 100644 index 0000000000000..189712f0fca78 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action.admin.indices.close; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class CloseIndexResponse extends ShardsAcknowledgedResponse { + + CloseIndexResponse() { + } + + public CloseIndexResponse(final boolean acknowledged, final boolean shardsAcknowledged) { + super(acknowledged, shardsAcknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + readShardsAcknowledged(in); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + writeShardsAcknowledged(out); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java index bb3db084b0c53..05f680af57ddf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.DestructiveOperations; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -44,7 +43,7 @@ /** * Close index action */ -public class TransportCloseIndexAction extends TransportMasterNodeAction { +public class TransportCloseIndexAction extends TransportMasterNodeAction { private final MetaDataIndexStateService indexStateService; private final DestructiveOperations destructiveOperations; @@ -76,12 +75,12 @@ protected String executor() { } @Override - protected AcknowledgedResponse newResponse() { - return new AcknowledgedResponse(); + protected CloseIndexResponse newResponse() { + return new CloseIndexResponse(); } @Override - protected void doExecute(Task task, CloseIndexRequest request, ActionListener listener) { + protected void doExecute(Task task, CloseIndexRequest request, ActionListener listener) { destructiveOperations.failDestructive(request.indices()); if (closeIndexEnabled == false) { throw new IllegalStateException("closing indices is disabled - set [" + CLUSTER_INDICES_CLOSE_ENABLE_SETTING.getKey() + @@ -97,29 +96,33 @@ protected ClusterBlockException checkBlock(CloseIndexRequest request, ClusterSta } @Override - protected void masterOperation(final CloseIndexRequest request, final ClusterState state, - final ActionListener listener) { + protected void masterOperation(final CloseIndexRequest request, + final ClusterState state, + final ActionListener listener) { throw new UnsupportedOperationException("The task parameter is required"); } @Override - protected void masterOperation(final Task task, final CloseIndexRequest request, final ClusterState state, - final ActionListener listener) throws Exception { + protected void masterOperation(final Task task, + final CloseIndexRequest request, + final ClusterState state, + final ActionListener listener) throws Exception { final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); if (concreteIndices == null || concreteIndices.length == 0) { - listener.onResponse(new AcknowledgedResponse(true)); + listener.onResponse(new CloseIndexResponse(true, false)); return; } final CloseIndexClusterStateUpdateRequest closeRequest = new CloseIndexClusterStateUpdateRequest(task.getId()) .ackTimeout(request.timeout()) .masterNodeTimeout(request.masterNodeTimeout()) + .waitForActiveShards(request.waitForActiveShards()) .indices(concreteIndices); - indexStateService.closeIndices(closeRequest, new ActionListener() { + indexStateService.closeIndices(closeRequest, new ActionListener() { @Override - public void onResponse(final AcknowledgedResponse response) { + public void onResponse(final CloseIndexResponse response) { listener.onResponse(response); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java index 8878713765ba0..78d2969c2fde4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.recovery; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.broadcast.BroadcastRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -47,7 +48,7 @@ public RecoveryRequest() { * @param indices Comma-separated list of indices about which to gather recovery information */ public RecoveryRequest(String... indices) { - super(indices); + super(indices, IndicesOptions.STRICT_EXPAND_OPEN_CLOSED); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java index f7356bd242d06..0ff31f42b9295 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java @@ -112,11 +112,11 @@ protected ShardsIterator shards(ClusterState state, RecoveryRequest request, Str @Override protected ClusterBlockException checkGlobalBlock(ClusterState state, RecoveryRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); } @Override protected ClusterBlockException checkRequestBlock(ClusterState state, RecoveryRequest request, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index c38c2a1c69a7f..81364c31c9ed9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -90,7 +90,11 @@ public enum Option { public static final IndicesOptions STRICT_EXPAND_OPEN = new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN)); public static final IndicesOptions LENIENT_EXPAND_OPEN = - new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE), EnumSet.of(WildcardStates.OPEN)); + new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE), + EnumSet.of(WildcardStates.OPEN)); + public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED = + new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE), + EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED)); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED = new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED)); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = @@ -440,6 +444,14 @@ public static IndicesOptions lenientExpandOpen() { return LENIENT_EXPAND_OPEN; } + /** + * @return indices options that ignores unavailable indices, expands wildcards to both open and closed + * indices and allows that no indices are resolved from wildcard expressions (not returning an error). + */ + public static IndicesOptions lenientExpand() { + return LENIENT_EXPAND_OPEN_CLOSED; + } + @Override public boolean equals(Object obj) { if (obj == null) { diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java index a04d2edc8dc63..6cf42e7ad3f14 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java @@ -40,6 +40,11 @@ protected BroadcastRequest(String[] indices) { this.indices = indices; } + protected BroadcastRequest(String[] indices, IndicesOptions indicesOptions) { + this.indices = indices; + this.indicesOptions = indicesOptions; + } + @Override public String[] indices() { return indices; diff --git a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index 718dde98a0f97..d5a73981f29f1 100644 --- a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -36,6 +36,7 @@ import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -307,7 +308,7 @@ public interface IndicesAdminClient extends ElasticsearchClient { * @return The result future * @see org.elasticsearch.client.Requests#closeIndexRequest(String) */ - ActionFuture close(CloseIndexRequest request); + ActionFuture close(CloseIndexRequest request); /** * Closes an index based on the index name. @@ -316,7 +317,7 @@ public interface IndicesAdminClient extends ElasticsearchClient { * @param listener A listener to be notified with a result * @see org.elasticsearch.client.Requests#closeIndexRequest(String) */ - void close(CloseIndexRequest request, ActionListener listener); + void close(CloseIndexRequest request, ActionListener listener); /** * Closes one or more indices based on their index name. diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 3fc931a85c0f7..e79f0567babe6 100644 --- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -152,6 +152,7 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexAction; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -1355,12 +1356,12 @@ public DeleteIndexRequestBuilder prepareDelete(String... indices) { } @Override - public ActionFuture close(final CloseIndexRequest request) { + public ActionFuture close(final CloseIndexRequest request) { return execute(CloseIndexAction.INSTANCE, request); } @Override - public void close(final CloseIndexRequest request, final ActionListener listener) { + public void close(final CloseIndexRequest request, final ActionListener listener) { execute(CloseIndexAction.INSTANCE, request, listener); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java index 0781cab1fe757..7c582483e3b42 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NotifyOnceListener; import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.close.TransportVerifyShardBeforeCloseAction; import org.elasticsearch.action.admin.indices.open.OpenIndexClusterStateUpdateRequest; import org.elasticsearch.action.support.ActiveShardsObserver; @@ -52,6 +53,8 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; @@ -90,6 +93,8 @@ public class MetaDataIndexStateService { public static final int INDEX_CLOSED_BLOCK_ID = 4; public static final ClusterBlock INDEX_CLOSED_BLOCK = new ClusterBlock(4, "index closed", false, false, false, RestStatus.FORBIDDEN, ClusterBlockLevel.READ_WRITE); + public static final Setting VERIFIED_BEFORE_CLOSE_SETTING = + Setting.boolSetting("index.verified_before_close", false, Setting.Property.IndexScope, Setting.Property.PrivateIndex); private final ClusterService clusterService; private final AllocationService allocationService; @@ -119,7 +124,7 @@ public MetaDataIndexStateService(ClusterService clusterService, AllocationServic * Closing indices is a 3 steps process: it first adds a write block to every indices to close, then waits for the operations on shards * to be terminated and finally closes the indices by moving their state to CLOSE. */ - public void closeIndices(final CloseIndexClusterStateUpdateRequest request, final ActionListener listener) { + public void closeIndices(final CloseIndexClusterStateUpdateRequest request, final ActionListener listener) { final Index[] concreteIndices = request.indices(); if (concreteIndices == null || concreteIndices.length == 0) { throw new IllegalArgumentException("Index name is required"); @@ -139,7 +144,7 @@ public ClusterState execute(final ClusterState currentState) { public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { if (oldState == newState) { assert blockedIndices.isEmpty() : "List of blocked indices is not empty but cluster state wasn't changed"; - listener.onResponse(new AcknowledgedResponse(true)); + listener.onResponse(new CloseIndexResponse(true, false)); } else { assert blockedIndices.isEmpty() == false : "List of blocked indices is empty but cluster state was changed"; threadPool.executor(ThreadPool.Names.MANAGEMENT) @@ -170,7 +175,29 @@ public void onFailure(final String source, final Exception e) { @Override public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { - listener.onResponse(new AcknowledgedResponse(acknowledged)); + + final String[] indices = results.entrySet().stream() + .filter(result -> result.getValue().isAcknowledged()) + .map(result -> result.getKey().getName()) + .filter(index -> newState.routingTable().hasIndex(index)) + .toArray(String[]::new); + + if (indices.length > 0) { + activeShardsObserver.waitForActiveShards(indices, request.waitForActiveShards(), + request.ackTimeout(), shardsAcknowledged -> { + if (shardsAcknowledged == false) { + logger.debug("[{}] indices closed, but the operation timed out while waiting " + + "for enough shards to be started.", Arrays.toString(indices)); + } + // acknowledged maybe be false but some indices may have been correctly closed, so + // we maintain a kind of coherency by overriding the shardsAcknowledged value + // (see ShardsAcknowledgedResponse constructor) + boolean shardsAcked = acknowledged ? shardsAcknowledged : false; + listener.onResponse(new CloseIndexResponse(acknowledged, shardsAcked)); + }, listener::onFailure); + } else { + listener.onResponse(new CloseIndexResponse(acknowledged, false)); + } } }), listener::onFailure) @@ -223,10 +250,6 @@ static ClusterState addIndexClosedBlocks(final Index[] indices, final Map blockedIndices, final Map results) { + + // Remove the index routing table of closed indices if the cluster is in a mixed version + // that does not support the replication of closed indices + final boolean removeRoutingTable = currentState.nodes().getMinNodeVersion().before(Version.V_8_0_0); + final MetaData.Builder metadata = MetaData.builder(currentState.metaData()); final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable()); @@ -409,15 +429,28 @@ static ClusterState closeRoutingTable(final ClusterState currentState, continue; } + blocks.removeIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID); + blocks.addIndexBlock(index.getName(), INDEX_CLOSED_BLOCK); + final IndexMetaData.Builder updatedMetaData = IndexMetaData.builder(indexMetaData).state(IndexMetaData.State.CLOSE); + if (removeRoutingTable) { + metadata.put(updatedMetaData); + routingTable.remove(index.getName()); + } else { + metadata.put(updatedMetaData + .settingsVersion(indexMetaData.getSettingsVersion() + 1) + .settings(Settings.builder() + .put(indexMetaData.getSettings()) + .put(VERIFIED_BEFORE_CLOSE_SETTING.getKey(), true))); + routingTable.addAsFromOpenToClose(metadata.getSafe(index)); + } + logger.debug("closing index {} succeeded", index); - blocks.removeIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID).addIndexBlock(index.getName(), INDEX_CLOSED_BLOCK); - metadata.put(IndexMetaData.builder(indexMetaData).state(IndexMetaData.State.CLOSE)); - routingTable.remove(index.getName()); closedIndices.add(index.getName()); } catch (final IndexNotFoundException e) { logger.debug("index {} has been deleted since it was blocked before closing, ignoring", index); } } + logger.info("completed closing of indices {}", closedIndices); return ClusterState.builder(currentState).blocks(blocks).metaData(metadata).routingTable(routingTable.build()).build(); } @@ -491,7 +524,15 @@ ClusterState openIndices(final Index[] indices, final ClusterState currentState) for (IndexMetaData indexMetaData : indicesToOpen) { final Index index = indexMetaData.getIndex(); if (indexMetaData.getState() != IndexMetaData.State.OPEN) { - IndexMetaData updatedIndexMetaData = IndexMetaData.builder(indexMetaData).state(IndexMetaData.State.OPEN).build(); + final Settings.Builder updatedSettings = Settings.builder().put(indexMetaData.getSettings()); + updatedSettings.remove(VERIFIED_BEFORE_CLOSE_SETTING.getKey()); + + IndexMetaData updatedIndexMetaData = IndexMetaData.builder(indexMetaData) + .state(IndexMetaData.State.OPEN) + .settingsVersion(indexMetaData.getSettingsVersion() + 1) + .settings(updatedSettings) + .build(); + // The index might be closed because we couldn't import it due to old incompatible version // We need to check that this index can be upgraded to the current version updatedIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(updatedIndexMetaData, minIndexCompatibilityVersion); @@ -555,4 +596,9 @@ public static ClusterBlock createIndexClosingBlock() { EnumSet.of(ClusterBlockLevel.WRITE)); } + public static boolean isIndexVerifiedBeforeClosed(final IndexMetaData indexMetaData) { + return indexMetaData.getState() == IndexMetaData.State.CLOSE + && VERIFIED_BEFORE_CLOSE_SETTING.exists(indexMetaData.getSettings()) + && VERIFIED_BEFORE_CLOSE_SETTING.get(indexMetaData.getSettings()); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index cf1235c8f2158..195ae2cce25b8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -358,6 +358,13 @@ public Builder initializeAsFromCloseToOpen(IndexMetaData indexMetaData) { return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, null)); } + /** + * Initializes a new empty index, as as a result of closing an opened index. + */ + public Builder initializeAsFromOpenToClose(IndexMetaData indexMetaData) { + return initializeEmpty(indexMetaData, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CLOSED, null)); + } + /** * Initializes a new empty index, to be restored from a snapshot */ diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index a6f7d58ce8589..3a49577563929 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -47,6 +47,8 @@ import java.util.Map; import java.util.function.Predicate; +import static org.elasticsearch.cluster.metadata.MetaDataIndexStateService.isIndexVerifiedBeforeClosed; + /** * Represents a global cluster-wide routing table for all indices including the * version of the current routing state. @@ -499,9 +501,9 @@ public Builder addAsNew(IndexMetaData indexMetaData) { } public Builder addAsRecovery(IndexMetaData indexMetaData) { - if (indexMetaData.getState() == IndexMetaData.State.OPEN) { + if (indexMetaData.getState() == IndexMetaData.State.OPEN || isIndexVerifiedBeforeClosed(indexMetaData)) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) - .initializeAsRecovery(indexMetaData); + .initializeAsRecovery(indexMetaData); add(indexRoutingBuilder); } return this; @@ -525,6 +527,13 @@ public Builder addAsFromCloseToOpen(IndexMetaData indexMetaData) { return this; } + public Builder addAsFromOpenToClose(IndexMetaData indexMetaData) { + assert isIndexVerifiedBeforeClosed(indexMetaData); + IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) + .initializeAsFromOpenToClose(indexMetaData); + return add(indexRoutingBuilder); + } + public Builder addAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) .initializeAsRestore(indexMetaData, recoverySource); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java index 3cad02367fdcf..de0ef52d7b78c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java @@ -118,7 +118,11 @@ public enum Reason { /** * Forced manually to allocate */ - MANUAL_ALLOCATION + MANUAL_ALLOCATION, + /** + * Unassigned as a result of closing an index. + */ + INDEX_CLOSED } /** @@ -264,6 +268,8 @@ public UnassignedInfo(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().before(Version.V_6_0_0_beta2) && reason == Reason.MANUAL_ALLOCATION) { out.writeByte((byte) Reason.ALLOCATION_FAILED.ordinal()); + } else if (out.getVersion().before(Version.V_7_0_0) && reason == Reason.INDEX_CLOSED) { + out.writeByte((byte) Reason.REINITIALIZED.ordinal()); } else { out.writeByte((byte) reason.ordinal()); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 9d936a28846e4..f1e42d2413815 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.settings; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; @@ -161,6 +162,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { EngineConfig.INDEX_CODEC_SETTING, IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS, IndexSettings.DEFAULT_PIPELINE, + MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index e3d8732c7fb81..51c625d334d55 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -673,7 +673,7 @@ public synchronized void updateMetaData(final IndexMetaData currentIndexMetaData // once we change the refresh interval we schedule yet another refresh // to ensure we are in a clean and predictable state. // it doesn't matter if we move from or to -1 in both cases we want - // docs to become visible immediately. This also flushes all pending indexing / search reqeusts + // docs to become visible immediately. This also flushes all pending indexing / search requests // that are waiting for a refresh. threadPool.executor(ThreadPool.Names.REFRESH).execute(new AbstractRunnable() { @Override @@ -830,17 +830,20 @@ private void sync(final Consumer sync, final String source) { } abstract static class BaseAsyncTask extends AbstractAsyncTask { + protected final IndexService indexService; - BaseAsyncTask(IndexService indexService, TimeValue interval) { + BaseAsyncTask(final IndexService indexService, final TimeValue interval) { super(indexService.logger, indexService.threadPool, interval, true); this.indexService = indexService; rescheduleIfNecessary(); } + @Override protected boolean mustReschedule() { - // don't re-schedule if its closed or if we don't have a single shard here..., we are done - return indexService.closed.get() == false; + // don't re-schedule if the IndexService instance is closed or if the index is closed + return indexService.closed.get() == false + && indexService.indexSettings.getIndexMetaData().getState() == IndexMetaData.State.OPEN; } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java new file mode 100644 index 0000000000000..fe1ad7a1a144f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.engine; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; + +import java.io.IOException; +import java.util.List; +import java.util.function.Function; + +/** + * NoOpEngine is an engine implementation that does nothing but the bare minimum + * required in order to have an engine. All attempts to do something (search, + * index, get), throw {@link UnsupportedOperationException}. + */ +public final class NoOpEngine extends ReadOnlyEngine { + + public NoOpEngine(EngineConfig config) { + super(config, null, null, true, Function.identity()); + } + + @Override + protected DirectoryReader open(final IndexCommit commit) throws IOException { + final Directory directory = commit.getDirectory(); + final List indexCommits = DirectoryReader.listCommits(directory); + final IndexCommit indexCommit = indexCommits.get(indexCommits.size() - 1); + return new DirectoryReader(directory, new LeafReader[0]) { + @Override + protected DirectoryReader doOpenIfChanged() throws IOException { + return null; + } + + @Override + protected DirectoryReader doOpenIfChanged(IndexCommit commit) throws IOException { + return null; + } + + @Override + protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws IOException { + return null; + } + + @Override + public long getVersion() { + return 0; + } + + @Override + public boolean isCurrent() throws IOException { + return true; + } + + @Override + public IndexCommit getIndexCommit() throws IOException { + return indexCommit; + } + + @Override + protected void doClose() throws IOException { + } + + @Override + public CacheHelper getReaderCacheHelper() { + return null; + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index f22bbbd471f69..d6e5449a80aa8 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -86,6 +86,7 @@ import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.engine.NoOpEngine; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; @@ -552,6 +553,12 @@ private synchronized IndexService createIndexService(final String reason, } private EngineFactory getEngineFactory(final IndexSettings idxSettings) { + final IndexMetaData indexMetaData = idxSettings.getIndexMetaData(); + if (indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE) { + // NoOpEngine takes precedence as long as the index is closed + return NoOpEngine::new; + } + final List> engineFactories = engineFactoryProviders .stream() diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index cceefc13d59dd..821e095fc20b0 100644 --- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -101,6 +101,7 @@ import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.FAILURE; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; +import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.REOPENED; public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateApplier { private static final Logger logger = LogManager.getLogger(IndicesClusterStateService.class); @@ -257,7 +258,7 @@ public synchronized void applyClusterState(final ClusterChangedEvent event) { deleteIndices(event); // also deletes shards of deleted indices - removeUnallocatedIndices(event); // also removes shards of removed indices + removeIndices(event); // also removes shards of removed indices failMissingShards(state); @@ -369,17 +370,18 @@ protected void doRun() throws Exception { } /** - * Removes indices that have no shards allocated to this node. This does not delete the shard data as we wait for enough - * shard copies to exist in the cluster before deleting shard data (triggered by {@link org.elasticsearch.indices.store.IndicesStore}). + * Removes indices that have no shards allocated to this node or indices whose state has changed. This does not delete the shard data + * as we wait for enough shard copies to exist in the cluster before deleting shard data (triggered by + * {@link org.elasticsearch.indices.store.IndicesStore}). * * @param event the cluster changed event */ - private void removeUnallocatedIndices(final ClusterChangedEvent event) { + private void removeIndices(final ClusterChangedEvent event) { final ClusterState state = event.state(); final String localNodeId = state.nodes().getLocalNodeId(); assert localNodeId != null; - Set indicesWithShards = new HashSet<>(); + final Set indicesWithShards = new HashSet<>(); RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId); if (localRoutingNode != null) { // null e.g. if we are not a data node for (ShardRouting shardRouting : localRoutingNode) { @@ -388,20 +390,27 @@ private void removeUnallocatedIndices(final ClusterChangedEvent event) { } for (AllocatedIndex indexService : indicesService) { - Index index = indexService.index(); - if (indicesWithShards.contains(index) == false) { + final Index index = indexService.index(); + final IndexMetaData indexMetaData = state.metaData().index(index); + final IndexMetaData existingMetaData = indexService.getIndexSettings().getIndexMetaData(); + + AllocatedIndices.IndexRemovalReason reason = null; + if (indexMetaData != null && indexMetaData.getState() != existingMetaData.getState()) { + reason = indexMetaData.getState() == IndexMetaData.State.CLOSE ? CLOSED : REOPENED; + } else if (indicesWithShards.contains(index) == false) { // if the cluster change indicates a brand new cluster, we only want // to remove the in-memory structures for the index and not delete the // contents on disk because the index will later be re-imported as a // dangling index - final IndexMetaData indexMetaData = state.metaData().index(index); assert indexMetaData != null || event.isNewCluster() : "index " + index + " does not exist in the cluster state, it should either " + "have been deleted or the cluster must be new"; - final AllocatedIndices.IndexRemovalReason reason = - indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE ? CLOSED : NO_LONGER_ASSIGNED; - logger.debug("{} removing index, [{}]", index, reason); - indicesService.removeIndex(index, reason, "removing index (no shards allocated)"); + reason = indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE ? CLOSED : NO_LONGER_ASSIGNED; + } + + if (reason != null) { + logger.debug("{} removing index ({})", index, reason); + indicesService.removeIndex(index, reason, "removing index (" + reason + ")"); } } } @@ -612,7 +621,7 @@ private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard ClusterState clusterState) { final ShardRouting currentRoutingEntry = shard.routingEntry(); assert currentRoutingEntry.isSameAllocation(shardRouting) : - "local shard has a different allocation id but wasn't cleaning by removeShards. " + "local shard has a different allocation id but wasn't cleaned by removeShards. " + "cluster state: " + shardRouting + " local: " + currentRoutingEntry; final long primaryTerm; @@ -747,7 +756,7 @@ private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFail private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure, ClusterState state) { try { logger.warn(() -> new ParameterizedMessage( - "[{}] marking and sending shard failed due to [{}]", shardRouting.shardId(), message), failure); + "{} marking and sending shard failed due to [{}]", shardRouting.shardId(), message), failure); failedShardsCache.put(shardRouting.shardId(), shardRouting); shardStateAction.localShardFailed(shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER, state); } catch (Exception inner) { @@ -948,7 +957,7 @@ enum IndexRemovalReason { DELETED, /** - * The index have been closed. The index should be removed and all associated resources released. Persistent parts of the index + * The index has been closed. The index should be removed and all associated resources released. Persistent parts of the index * like the shards files, state and transaction logs are kept around in the case of a disaster recovery. */ CLOSED, @@ -958,7 +967,13 @@ enum IndexRemovalReason { * Persistent parts of the index like the shards files, state and transaction logs are kept around in the * case of a disaster recovery. */ - FAILURE + FAILURE, + + /** + * The index has been reopened. The index should be removed and all associated resources released. Persistent parts of the index + * like the shards files, state and transaction logs are kept around in the case of a disaster recovery. + */ + REOPENED, } } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java index 2f68cac9a0e1f..5e6f98eedd853 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; @@ -39,9 +40,9 @@ import static org.elasticsearch.client.Requests.clusterHealthRequest; public class RestClusterHealthAction extends BaseRestHandler { + public RestClusterHealthAction(Settings settings, RestController controller) { super(settings); - controller.registerHandler(RestRequest.Method.GET, "/_cluster/health", this); controller.registerHandler(RestRequest.Method.GET, "/_cluster/health/{index}", this); } @@ -53,7 +54,8 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - ClusterHealthRequest clusterHealthRequest = clusterHealthRequest(Strings.splitStringByCommaToArray(request.param("index"))); + final ClusterHealthRequest clusterHealthRequest = clusterHealthRequest(Strings.splitStringByCommaToArray(request.param("index"))); + clusterHealthRequest.indicesOptions(IndicesOptions.fromRequest(request, clusterHealthRequest.indicesOptions())); clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local())); clusterHealthRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterHealthRequest.masterNodeTimeout())); clusterHealthRequest.timeout(request.paramAsTime("timeout", clusterHealthRequest.timeout())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java index b2475cafcbeb6..3ee2687eb7288 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.rest.action.admin.indices; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; @@ -49,6 +50,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC closeIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", closeIndexRequest.masterNodeTimeout())); closeIndexRequest.timeout(request.paramAsTime("timeout", closeIndexRequest.timeout())); closeIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, closeIndexRequest.indicesOptions())); + String waitForActiveShards = request.param("wait_for_active_shards"); + if (waitForActiveShards != null) { + closeIndexRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); + } return channel -> client.admin().indices().close(closeIndexRequest, new RestToXContentListener<>(channel)); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index 9b672a9992cc8..24f3ac8522aa2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -104,9 +104,9 @@ public void processResponse(final ClusterStateResponse clusterStateResponse) { // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests not to // fail on the deleted index (as we want to ignore wildcards that cannot be resolved). // This behavior can be ensured by letting the cluster health and indices stats requests re-resolve the index names with the - // same indices options that we used for the initial cluster state request (strictExpand). Unfortunately cluster health - // requests hard-code their indices options and the best we can do is apply strictExpand to the indices stats request. + // same indices options that we used for the initial cluster state request (strictExpand). final ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(indices); + clusterHealthRequest.indicesOptions(strictExpandIndicesOptions); clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local())); client.admin().cluster().health(clusterHealthRequest, new RestActionListener(channel) { @@ -383,34 +383,37 @@ protected Table getTableWithHeader(final RestRequest request) { } // package private for testing - Table buildTable(RestRequest request, IndexMetaData[] indicesMetaData, ClusterHealthResponse response, IndicesStatsResponse stats) { + Table buildTable(final RestRequest request, + final IndexMetaData[] indicesMetaData, + final ClusterHealthResponse clusterHealthResponse, + final IndicesStatsResponse indicesStatsResponse) { final String healthParam = request.param("health"); - final ClusterHealthStatus status; - if (healthParam != null) { - status = ClusterHealthStatus.fromString(healthParam); - } else { - status = null; - } - - Table table = getTableWithHeader(request); + final Table table = getTableWithHeader(request); for (IndexMetaData indexMetaData : indicesMetaData) { final String indexName = indexMetaData.getIndex().getName(); - ClusterIndexHealth indexHealth = response.getIndices().get(indexName); - IndexStats indexStats = stats.getIndices().get(indexName); - IndexMetaData.State state = indexMetaData.getState(); - boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(indexMetaData.getSettings()); - - if (status != null) { - if (state == IndexMetaData.State.CLOSE || - (indexHealth == null && false == ClusterHealthStatus.RED.equals(status)) || - false == indexHealth.getStatus().equals(status)) { + final ClusterIndexHealth indexHealth = clusterHealthResponse.getIndices().get(indexName); + final IndexStats indexStats = indicesStatsResponse.getIndices().get(indexName); + final IndexMetaData.State indexState = indexMetaData.getState(); + final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(indexMetaData.getSettings()); + + if (healthParam != null) { + final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); + boolean skip; + if (indexHealth != null) { + // index health is known but does not match the one requested + skip = indexHealth.getStatus() != healthStatusFilter; + } else { + // index health is unknown, skip if we don't explicitly request RED health or if the index is closed but not replicated + skip = ClusterHealthStatus.RED != healthStatusFilter || indexState == IndexMetaData.State.CLOSE; + } + if (skip) { continue; } } // the open index is present in the cluster state but is not returned in the indices stats API - if (indexStats == null && state != IndexMetaData.State.CLOSE) { + if (indexStats == null && indexState != IndexMetaData.State.CLOSE) { // the index stats API is called last, after cluster state and cluster health. If the index stats // has not resolved the same open indices as the initial cluster state call, then the indices might // have been removed in the meantime or, more likely, are unauthorized. This is because the cluster @@ -422,9 +425,8 @@ Table buildTable(RestRequest request, IndexMetaData[] indicesMetaData, ClusterHe final CommonStats primaryStats; final CommonStats totalStats; - if (state == IndexMetaData.State.CLOSE) { + if (indexState == IndexMetaData.State.CLOSE) { // empty stats for closed indices, but their names are displayed - assert indexStats == null; primaryStats = new CommonStats(); totalStats = new CommonStats(); } else { @@ -433,9 +435,15 @@ Table buildTable(RestRequest request, IndexMetaData[] indicesMetaData, ClusterHe } table.startRow(); - table.addCell(state == IndexMetaData.State.OPEN ? - (indexHealth == null ? "red*" : indexHealth.getStatus().toString().toLowerCase(Locale.ROOT)) : null); - table.addCell(state.toString().toLowerCase(Locale.ROOT)); + + String health = null; + if (indexHealth != null) { + health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); + } else if (indexStats != null) { + health = "red*"; + } + table.addCell(health); + table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); table.addCell(indexName); table.addCell(indexMetaData.getIndexUUID()); table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 06968250e9c88..5ad15fa626822 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -268,7 +268,7 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem // it's fine to keep the contexts open if the index is still "alive" // unfortunately we don't have a clear way to signal today why an index is closed. // to release memory and let references to the filesystem go etc. - if (reason == IndexRemovalReason.DELETED || reason == IndexRemovalReason.CLOSED) { + if (reason == IndexRemovalReason.DELETED || reason == IndexRemovalReason.CLOSED || reason == IndexRemovalReason.REOPENED) { freeAllContextForIndex(index); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java index e38fc64c8e3ab..941ad3c658aba 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java @@ -19,9 +19,12 @@ package org.elasticsearch.action.admin.cluster.allocation; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -32,8 +35,10 @@ import org.elasticsearch.cluster.routing.allocation.MoveDecision; import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult; import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -52,6 +57,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -70,8 +76,7 @@ public void testUnassignedPrimaryWithExistingIndex() throws Exception { logger.info("--> starting 2 nodes"); internalCluster().startNodes(2); - logger.info("--> creating an index with 1 primary, 0 replicas"); - createIndexAndIndexData(1, 0); + prepareIndex(1, 0); logger.info("--> stopping the node with the primary"); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName())); @@ -149,8 +154,7 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception { logger.info("--> starting 3 nodes"); internalCluster().startNodes(3); - logger.info("--> creating an index with 1 primary, 1 replica"); - createIndexAndIndexData(1, 1); + prepareIndex(1, 1); logger.info("--> stopping the node with the replica"); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode().getName())); ensureStableCluster(2); @@ -268,8 +272,7 @@ public void testUnassignedReplicaWithPriorCopy() throws Exception { logger.info("--> starting 3 nodes"); List nodes = internalCluster().startNodes(3); - logger.info("--> creating an index with 1 primary and 1 replica"); - createIndexAndIndexData(1, 1); + prepareIndex(1, 1); String primaryNodeName = primaryNodeName(); nodes.remove(primaryNodeName); @@ -390,7 +393,8 @@ public void testAllocationFilteringOnIndexCreation() throws Exception { internalCluster().startNodes(2); logger.info("--> creating an index with 1 primary, 0 replicas, with allocation filtering so the primary can't be assigned"); - createIndexAndIndexData(1, 0, Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(), + prepareIndex(IndexMetaData.State.OPEN, 1, 0, + Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(), ActiveShardCount.NONE); boolean includeYesDecisions = randomBoolean(); @@ -481,8 +485,7 @@ public void testAllocationFilteringPreventsShardMove() throws Exception { logger.info("--> starting 2 nodes"); internalCluster().startNodes(2); - logger.info("--> creating an index with 1 primary and 0 replicas"); - createIndexAndIndexData(1, 0); + prepareIndex(1, 0); logger.info("--> setting up allocation filtering to prevent allocation to both nodes"); client().admin().indices().prepareUpdateSettings("idx").setSettings( @@ -591,8 +594,7 @@ public void testRebalancingNotAllowed() throws Exception { internalCluster().startNode(); ensureStableCluster(1); - logger.info("--> creating an index with 5 shards, all allocated to the single node"); - createIndexAndIndexData(5, 0); + prepareIndex(5, 0); logger.info("--> disabling rebalancing on the index"); client().admin().indices().prepareUpdateSettings("idx").setSettings( @@ -704,8 +706,7 @@ public void testWorseBalance() throws Exception { internalCluster().startNode(); ensureStableCluster(1); - logger.info("--> creating an index with 5 shards, all allocated to the single node"); - createIndexAndIndexData(5, 0); + prepareIndex(5, 0); logger.info("--> setting balancing threshold really high, so it won't be met"); client().admin().cluster().prepareUpdateSettings().setTransientSettings( @@ -808,8 +809,7 @@ public void testBetterBalanceButCannotAllocate() throws Exception { String firstNode = internalCluster().startNode(); ensureStableCluster(1); - logger.info("--> creating an index with 5 shards, all allocated to the single node"); - createIndexAndIndexData(5, 0); + prepareIndex(5, 0); logger.info("--> setting up allocation filtering to only allow allocation to the current node"); client().admin().indices().prepareUpdateSettings("idx").setSettings( @@ -918,9 +918,9 @@ public void testAssignedReplicaOnSpecificNode() throws Exception { logger.info("--> starting 3 nodes"); List nodes = internalCluster().startNodes(3); - logger.info("--> creating an index with 1 primary and 2 replicas"); String excludedNode = nodes.get(randomIntBetween(0, 2)); - createIndexAndIndexData(1, 2, Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(), + prepareIndex(randomIndexState(), 1, 2, + Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(), ActiveShardCount.from(2)); boolean includeYesDecisions = randomBoolean(); @@ -1019,8 +1019,7 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception { final String replicaNode = internalCluster().startNode(); final String primaryNode = internalCluster().startNode(); - logger.info("--> creating an index with 1 primary and 1 replica"); - createIndexAndIndexData(1, 1, + prepareIndex(IndexMetaData.State.OPEN, 1, 1, Settings.builder() .put("index.routing.allocation.include._name", primaryNode) .put("index.routing.allocation.exclude._name", masterNode) @@ -1037,8 +1036,22 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception { logger.info("--> stop node with the replica shard"); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode)); - logger.info("--> index more data, now the replica is stale"); - indexData(); + final IndexMetaData.State indexState = randomIndexState(); + if (indexState == IndexMetaData.State.OPEN) { + logger.info("--> index more data, now the replica is stale"); + indexData(); + } else { + logger.info("--> close the index, now the replica is stale"); + assertAcked(client().admin().indices().prepareClose("idx")); + + final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx") + .setTimeout(TimeValue.timeValueSeconds(30)) + .setWaitForActiveShards(ActiveShardCount.ONE) + .setWaitForNoInitializingShards(true) + .setWaitForEvents(Priority.LANGUID) + .get(); + assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value())); + } logger.info("--> stop the node with the primary"); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); @@ -1147,21 +1160,39 @@ private ClusterAllocationExplanation runExplain(boolean primary, String nodeId, return explanation; } - private void createIndexAndIndexData(int numPrimaries, int numReplicas) { - createIndexAndIndexData(numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL); + private void prepareIndex(final int numPrimaries, final int numReplicas) { + prepareIndex(randomIndexState(), numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL); } - private void createIndexAndIndexData(int numPrimaries, int numReplicas, Settings settings, ActiveShardCount activeShardCount) { - client().admin().indices().prepareCreate("idx") + private void prepareIndex(final IndexMetaData.State state, final int numPrimaries, final int numReplicas, + final Settings settings, final ActiveShardCount activeShardCount) { + + logger.info("--> creating a {} index with {} primary, {} replicas", state, numPrimaries, numReplicas); + assertAcked(client().admin().indices().prepareCreate("idx") .setSettings(Settings.builder() .put("index.number_of_shards", numPrimaries) .put("index.number_of_replicas", numReplicas) .put(settings)) .setWaitForActiveShards(activeShardCount) - .get(); + .get()); + if (activeShardCount != ActiveShardCount.NONE) { indexData(); } + if (state == IndexMetaData.State.CLOSE) { + assertAcked(client().admin().indices().prepareClose("idx")); + + final ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth("idx") + .setTimeout(TimeValue.timeValueSeconds(30)) + .setWaitForActiveShards(activeShardCount) + .setWaitForEvents(Priority.LANGUID) + .get(); + assertThat(clusterHealthResponse.getStatus().value(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW.value())); + } + } + + private static IndexMetaData.State randomIndexState() { + return randomFrom(IndexMetaData.State.values()); } private void indexData() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java index 1e85005add0e9..75fa5a2393d37 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java @@ -19,15 +19,23 @@ package org.elasticsearch.action.admin.cluster.health; +import org.elasticsearch.Version; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; +import java.util.Locale; + +import static org.elasticsearch.test.VersionUtils.getPreviousVersion; +import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.core.IsEqual.equalTo; public class ClusterHealthRequestTests extends ESTestCase { + public void testSerialize() throws Exception { final ClusterHealthRequest originalRequest = randomRequest(); final ClusterHealthRequest cloneRequest; @@ -43,9 +51,89 @@ public void testSerialize() throws Exception { assertThat(cloneRequest.waitForNoRelocatingShards(), equalTo(originalRequest.waitForNoRelocatingShards())); assertThat(cloneRequest.waitForActiveShards(), equalTo(originalRequest.waitForActiveShards())); assertThat(cloneRequest.waitForEvents(), equalTo(originalRequest.waitForEvents())); + assertIndicesEquals(cloneRequest.indices(), originalRequest.indices()); + assertThat(cloneRequest.indicesOptions(), equalTo(originalRequest.indicesOptions())); + } + + public void testBwcSerialization() throws Exception { + for (int runs = 0; runs < randomIntBetween(5, 20); runs++) { + // Generate a random cluster health request in version < 8.0.0 and serializes it + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(randomVersionBetween(random(), Version.V_6_3_0, getPreviousVersion(Version.V_8_0_0))); + + final ClusterHealthRequest expected = randomRequest(); + { + expected.getParentTask().writeTo(out); + out.writeTimeValue(expected.masterNodeTimeout()); + out.writeBoolean(expected.local()); + if (expected.indices() == null) { + out.writeVInt(0); + } else { + out.writeVInt(expected.indices().length); + for (String index : expected.indices()) { + out.writeString(index); + } + } + out.writeTimeValue(expected.timeout()); + if (expected.waitForStatus() == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeByte(expected.waitForStatus().value()); + } + out.writeBoolean(expected.waitForNoRelocatingShards()); + expected.waitForActiveShards().writeTo(out); + out.writeString(expected.waitForNodes()); + if (expected.waitForEvents() == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + Priority.writeTo(expected.waitForEvents(), out); + } + out.writeBoolean(expected.waitForNoInitializingShards()); + } + + // Deserialize and check the cluster health request + final StreamInput in = out.bytes().streamInput(); + in.setVersion(out.getVersion()); + final ClusterHealthRequest actual = new ClusterHealthRequest(in); + + assertThat(actual.waitForStatus(), equalTo(expected.waitForStatus())); + assertThat(actual.waitForNodes(), equalTo(expected.waitForNodes())); + assertThat(actual.waitForNoInitializingShards(), equalTo(expected.waitForNoInitializingShards())); + assertThat(actual.waitForNoRelocatingShards(), equalTo(expected.waitForNoRelocatingShards())); + assertThat(actual.waitForActiveShards(), equalTo(expected.waitForActiveShards())); + assertThat(actual.waitForEvents(), equalTo(expected.waitForEvents())); + assertIndicesEquals(actual.indices(), expected.indices()); + assertThat(actual.indicesOptions(), equalTo(IndicesOptions.lenientExpandOpen())); + } + + for (int runs = 0; runs < randomIntBetween(5, 20); runs++) { + // Generate a random cluster health request in current version + final ClusterHealthRequest expected = randomRequest(); + + // Serialize to node in version < 8.0.0 + final BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(randomVersionBetween(random(), Version.V_6_3_0, getPreviousVersion(Version.V_8_0_0))); + expected.writeTo(out); + + // Deserialize and check the cluster health request + final StreamInput in = out.bytes().streamInput(); + in.setVersion(out.getVersion()); + final ClusterHealthRequest actual = new ClusterHealthRequest(in); + + assertThat(actual.waitForStatus(), equalTo(expected.waitForStatus())); + assertThat(actual.waitForNodes(), equalTo(expected.waitForNodes())); + assertThat(actual.waitForNoInitializingShards(), equalTo(expected.waitForNoInitializingShards())); + assertThat(actual.waitForNoRelocatingShards(), equalTo(expected.waitForNoRelocatingShards())); + assertThat(actual.waitForActiveShards(), equalTo(expected.waitForActiveShards())); + assertThat(actual.waitForEvents(), equalTo(expected.waitForEvents())); + assertIndicesEquals(actual.indices(), expected.indices()); + assertThat(actual.indicesOptions(), equalTo(IndicesOptions.lenientExpandOpen())); + } } - ClusterHealthRequest randomRequest() { + private ClusterHealthRequest randomRequest() { ClusterHealthRequest request = new ClusterHealthRequest(); request.waitForStatus(randomFrom(ClusterHealthStatus.values())); request.waitForNodes(randomFrom("", "<", "<=", ">", ">=") + between(0, 1000)); @@ -53,7 +141,21 @@ ClusterHealthRequest randomRequest() { request.waitForNoRelocatingShards(randomBoolean()); request.waitForActiveShards(randomIntBetween(0, 10)); request.waitForEvents(randomFrom(Priority.values())); + if (randomBoolean()) { + final String[] indices = new String[randomIntBetween(1, 10)]; + for (int i = 0; i < indices.length; i++) { + indices[i] = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + } + request.indices(indices); + } + if (randomBoolean()) { + request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + } return request; } + private static void assertIndicesEquals(final String[] actual, final String[] expected) { + // null indices in ClusterHealthRequest is deserialized as empty string array + assertArrayEquals(expected != null ? expected : Strings.EMPTY_ARRAY, actual); + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java new file mode 100644 index 0000000000000..53b39027a697b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.close; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +import static org.elasticsearch.test.VersionUtils.randomVersionBetween; + +public class CloseIndexRequestTests extends ESTestCase { + + public void testSerialization() throws Exception { + final CloseIndexRequest request = randomRequest(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + request.writeTo(out); + + final CloseIndexRequest deserializedRequest = new CloseIndexRequest(); + try (StreamInput in = out.bytes().streamInput()) { + deserializedRequest.readFrom(in); + } + assertEquals(request.timeout(), deserializedRequest.timeout()); + assertEquals(request.masterNodeTimeout(), deserializedRequest.masterNodeTimeout()); + assertEquals(request.indicesOptions(), deserializedRequest.indicesOptions()); + assertEquals(request.getParentTask(), deserializedRequest.getParentTask()); + assertEquals(request.waitForActiveShards(), deserializedRequest.waitForActiveShards()); + assertArrayEquals(request.indices(), deserializedRequest.indices()); + } + } + + public void testBwcSerialization() throws Exception { + { + final CloseIndexRequest request = randomRequest(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(randomVersionBetween(random(), Version.V_6_4_0, VersionUtils.getPreviousVersion(Version.V_8_0_0))); + request.writeTo(out); + + try (StreamInput in = out.bytes().streamInput()) { + assertEquals(request.getParentTask(), TaskId.readFromStream(in)); + assertEquals(request.masterNodeTimeout(), in.readTimeValue()); + assertEquals(request.timeout(), in.readTimeValue()); + assertArrayEquals(request.indices(), in.readStringArray()); + assertEquals(request.indicesOptions(), IndicesOptions.readIndicesOptions(in)); + } + } + } + { + final CloseIndexRequest sample = randomRequest(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + sample.getParentTask().writeTo(out); + out.writeTimeValue(sample.masterNodeTimeout()); + out.writeTimeValue(sample.timeout()); + out.writeStringArray(sample.indices()); + sample.indicesOptions().writeIndicesOptions(out); + + final CloseIndexRequest deserializedRequest = new CloseIndexRequest(); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(randomVersionBetween(random(), Version.V_6_4_0, VersionUtils.getPreviousVersion(Version.V_8_0_0))); + deserializedRequest.readFrom(in); + } + assertEquals(sample.getParentTask(), deserializedRequest.getParentTask()); + assertEquals(sample.masterNodeTimeout(), deserializedRequest.masterNodeTimeout()); + assertEquals(sample.timeout(), deserializedRequest.timeout()); + assertArrayEquals(sample.indices(), deserializedRequest.indices()); + assertEquals(sample.indicesOptions(), deserializedRequest.indicesOptions()); + assertEquals(ActiveShardCount.NONE, deserializedRequest.waitForActiveShards()); + } + } + } + + private CloseIndexRequest randomRequest() { + CloseIndexRequest request = new CloseIndexRequest(); + request.indices(generateRandomStringArray(10, 5, false, false)); + if (randomBoolean()) { + request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + } + if (randomBoolean()) { + request.timeout(randomPositiveTimeValue()); + } + if (randomBoolean()) { + request.masterNodeTimeout(randomPositiveTimeValue()); + } + if (randomBoolean()) { + request.setParentTask(randomAlphaOfLength(5), randomNonNegativeLong()); + } + if (randomBoolean()) { + request.waitForActiveShards(randomFrom(ActiveShardCount.DEFAULT, ActiveShardCount.NONE, ActiveShardCount.ONE, + ActiveShardCount.ALL)); + } + return request; + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java new file mode 100644 index 0000000000000..dc859cfab63a9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.close; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +import static org.elasticsearch.test.VersionUtils.randomVersionBetween; +import static org.hamcrest.Matchers.equalTo; + +public class CloseIndexResponseTests extends ESTestCase { + + public void testSerialization() throws Exception { + final CloseIndexResponse response = randomResponse(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + response.writeTo(out); + + final CloseIndexResponse deserializedResponse = new CloseIndexResponse(); + try (StreamInput in = out.bytes().streamInput()) { + deserializedResponse.readFrom(in); + } + assertCloseIndexResponse(deserializedResponse, response); + } + } + + public void testBwcSerialization() throws Exception { + { + final CloseIndexResponse response = randomResponse(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_8_0_0))); + response.writeTo(out); + + final AcknowledgedResponse deserializedResponse = new AcknowledgedResponse(); + try (StreamInput in = out.bytes().streamInput()) { + deserializedResponse.readFrom(in); + } + assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged())); + } + } + { + final AcknowledgedResponse response = new AcknowledgedResponse(randomBoolean()); + try (BytesStreamOutput out = new BytesStreamOutput()) { + response.writeTo(out); + + final CloseIndexResponse deserializedResponse = new CloseIndexResponse(); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_8_0_0))); + deserializedResponse.readFrom(in); + } + assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged())); + } + } + } + + private CloseIndexResponse randomResponse() { + final boolean acknowledged = randomBoolean(); + final boolean shardsAcknowledged = acknowledged ? randomBoolean() : false; + return new CloseIndexResponse(acknowledged, shardsAcknowledged); + } + + private static void assertCloseIndexResponse(final CloseIndexResponse actual, final CloseIndexResponse expected) { + assertThat(actual.isAcknowledged(), equalTo(expected.isAcknowledged())); + assertThat(actual.isShardsAcknowledged(), equalTo(expected.isShardsAcknowledged())); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java index 8693308e8650b..d0680e91b8b73 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterHealthIT.java @@ -20,13 +20,19 @@ package org.elasticsearch.cluster; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; + import java.util.concurrent.atomic.AtomicBoolean; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; public class ClusterHealthIT extends ESIntegTestCase { @@ -76,6 +82,159 @@ public void testHealth() { assertThat(healthResponse.getIndices().size(), equalTo(1)); } + public void testHealthWithClosedIndices() { + createIndex("index-1"); + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForGreenStatus().get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + + createIndex("index-2"); + assertAcked(client().admin().indices().prepareClose("index-2")); + + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth() + .setWaitForGreenStatus() + .get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(2)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-1").get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-2").get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-*").get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(2)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-*") + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-2"), nullValue()); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-*") + .setIndicesOptions(IndicesOptions.fromOptions(true, true, false, true)) + .get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-1"), nullValue()); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + + createIndex("index-3", Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 50) + .build()); + assertAcked(client().admin().indices().prepareClose("index-3")); + + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth() + .setWaitForNoRelocatingShards(true) + .setWaitForNoInitializingShards(true) + .setWaitForYellowStatus() + .get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(3)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-3").getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-1").get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-2").get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-3").get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-3").getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-*").get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(3)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-3").getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-*") + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(1)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-2"), nullValue()); + assertThat(response.getIndices().get("index-3"), nullValue()); + } + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth("index-*") + .setIndicesOptions(IndicesOptions.fromOptions(true, true, false, true)) + .get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(2)); + assertThat(response.getIndices().get("index-1"), nullValue()); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-3").getStatus(), equalTo(ClusterHealthStatus.YELLOW)); + } + + assertAcked(client().admin().indices().prepareUpdateSettings("index-3") + .setSettings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas()) + .build())); + { + ClusterHealthResponse response = client().admin().cluster().prepareHealth() + .setWaitForGreenStatus() + .get(); + assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.isTimedOut(), equalTo(false)); + assertThat(response.getIndices().size(), equalTo(3)); + assertThat(response.getIndices().get("index-1").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-2").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.getIndices().get("index-3").getStatus(), equalTo(ClusterHealthStatus.GREEN)); + } + } + public void testHealthOnIndexCreation() throws Exception { final AtomicBoolean finished = new AtomicBoolean(false); Thread clusterHealthThread = new Thread() { diff --git a/server/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java index 083c914b37052..edcf4446dc2bf 100644 --- a/server/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java @@ -24,6 +24,8 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetaData.State; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -33,9 +35,11 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; @@ -54,7 +58,6 @@ public void testSimpleAwareness() throws Exception { .put("cluster.routing.allocation.awareness.attributes", "rack_id") .build(); - logger.info("--> starting 2 nodes on the same rack"); internalCluster().startNodes(2, Settings.builder().put(commonSettings).put("node.attr.rack_id", "rack_1").build()); @@ -68,6 +71,9 @@ public void testSimpleAwareness() throws Exception { ensureGreen(); + final List indicesToClose = randomSubsetOf(Arrays.asList("test1", "test2")); + indicesToClose.forEach(indexToClose -> assertAcked(client().admin().indices().prepareClose(indexToClose).get())); + logger.info("--> starting 1 node on a different rack"); final String node3 = internalCluster().startNode(Settings.builder().put(commonSettings).put("node.attr.rack_id", "rack_2").build()); @@ -75,14 +81,23 @@ public void testSimpleAwareness() throws Exception { assertThat(awaitBusy( () -> { logger.info("--> waiting for no relocation"); - ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID) - .setWaitForGreenStatus().setWaitForNodes("3").setWaitForNoRelocatingShards(true).get(); + ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth() + .setIndices("test1", "test2") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNodes("3") + .setWaitForNoRelocatingShards(true) + .get(); if (clusterHealth.isTimedOut()) { return false; } logger.info("--> checking current state"); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); + // check that closed indices are effectively closed + if (indicesToClose.stream().anyMatch(index -> clusterState.metaData().index(index).getState() != State.CLOSE)) { + return false; + } // verify that we have all the primaries on node3 ObjectIntHashMap counts = new ObjectIntHashMap<>(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { @@ -99,7 +114,7 @@ public void testSimpleAwareness() throws Exception { ), equalTo(true)); } - public void testAwarenessZones() throws Exception { + public void testAwarenessZones() { Settings commonSettings = Settings.builder() .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a,b") .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone") @@ -121,12 +136,20 @@ public void testAwarenessZones() throws Exception { ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("4").execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); - client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put("index.number_of_shards", 5) - .put("index.number_of_replicas", 1)).execute().actionGet(); + createIndex("test", Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .build()); + + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test")); + } logger.info("--> waiting for shards to be allocated"); - health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus() + health = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() .setWaitForNoRelocatingShards(true).execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); @@ -146,7 +169,7 @@ public void testAwarenessZones() throws Exception { assertThat(counts.get(B_0), anyOf(equalTo(2),equalTo(3))); } - public void testAwarenessZonesIncrementalNodes() throws Exception { + public void testAwarenessZonesIncrementalNodes() { Settings commonSettings = Settings.builder() .put("cluster.routing.allocation.awareness.force.zone.values", "a,b") .put("cluster.routing.allocation.awareness.attributes", "zone") @@ -159,11 +182,23 @@ public void testAwarenessZonesIncrementalNodes() throws Exception { ); String A_0 = nodes.get(0); String B_0 = nodes.get(1); - client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put("index.number_of_shards", 5) - .put("index.number_of_replicas", 1)).execute().actionGet(); - ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID) - .setWaitForGreenStatus().setWaitForNodes("2").setWaitForNoRelocatingShards(true).execute().actionGet(); + + createIndex("test", Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .build()); + + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test")); + } + + ClusterHealthResponse health = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNodes("2") + .setWaitForNoRelocatingShards(true) + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); ObjectIntHashMap counts = new ObjectIntHashMap<>(); @@ -180,12 +215,22 @@ public void testAwarenessZonesIncrementalNodes() throws Exception { logger.info("--> starting another node in zone 'b'"); String B_1 = internalCluster().startNode(Settings.builder().put(commonSettings).put("node.attr.zone", "b").build()); - health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus() - .setWaitForNodes("3").execute().actionGet(); + health = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNodes("3") + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); client().admin().cluster().prepareReroute().get(); - health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus() - .setWaitForNodes("3").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet(); + health = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNodes("3") + .setWaitForActiveShards(10) + .setWaitForNoRelocatingShards(true) + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); @@ -204,12 +249,22 @@ public void testAwarenessZonesIncrementalNodes() throws Exception { assertThat(counts.get(B_1), equalTo(2)); String noZoneNode = internalCluster().startNode(); - health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus() - .setWaitForNodes("4").execute().actionGet(); + health = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNodes("4") + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); client().admin().cluster().prepareReroute().get(); - health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus() - .setWaitForNodes("4").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet(); + health = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNodes("4") + .setWaitForActiveShards(10) + .setWaitForNoRelocatingShards(true) + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); @@ -231,8 +286,14 @@ public void testAwarenessZonesIncrementalNodes() throws Exception { client().admin().cluster().prepareUpdateSettings() .setTransientSettings(Settings.builder().put("cluster.routing.allocation.awareness.attributes", "").build()).get(); - health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus() - .setWaitForNodes("4").setWaitForActiveShards(10).setWaitForNoRelocatingShards(true).execute().actionGet(); + health = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNodes("4") + .setWaitForActiveShards(10) + .setWaitForNoRelocatingShards(true) + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); diff --git a/server/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/server/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java index 71c9f5a15ba4d..d629804b02d58 100644 --- a/server/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java @@ -22,7 +22,6 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; @@ -34,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.RerouteExplanation; import org.elasticsearch.cluster.routing.allocation.RoutingExplanations; import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand; @@ -48,6 +48,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; @@ -102,6 +103,10 @@ private void rerouteWithCommands(Settings commonSettings) throws Exception { .setSettings(Settings.builder().put("index.number_of_shards", 1)) .execute().actionGet(); + if (randomBoolean()) { + client().admin().indices().prepareClose("test").get(); + } + ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.getRoutingNodes().unassigned().size(), equalTo(2)); @@ -128,8 +133,11 @@ private void rerouteWithCommands(Settings commonSettings) throws Exception { assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING)); - ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID) - .setWaitForYellowStatus().execute().actionGet(); + ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForYellowStatus() + .execute().actionGet(); assertThat(healthResponse.isTimedOut(), equalTo(false)); logger.info("--> get the state, verify shard 1 primary allocated"); @@ -149,9 +157,12 @@ private void rerouteWithCommands(Settings commonSettings) throws Exception { assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_2).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING)); - - healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus() - .setWaitForNoRelocatingShards(true).execute().actionGet(); + healthResponse = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForYellowStatus() + .setWaitForNoRelocatingShards(true) + .execute().actionGet(); assertThat(healthResponse.isTimedOut(), equalTo(false)); logger.info("--> get the state, verify shard 1 primary moved from node1 to node2"); @@ -193,11 +204,15 @@ public void testDelayWithALargeAmountOfShards() throws Exception { logger.info("--> create indices"); for (int i = 0; i < 25; i++) { - client().admin().indices().prepareCreate("test" + i) - .setSettings(Settings.builder() - .put("index.number_of_shards", 5).put("index.number_of_replicas", 1) - .put("index.unassigned.node_left.delayed_timeout", randomIntBetween(250, 1000) + "ms")) - .execute().actionGet(); + final String indexName = "test" + i; + createIndex(indexName, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), randomIntBetween(250, 1000) + "ms") + .build()); + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose(indexName)); + } } ensureGreen(TimeValue.timeValueMinutes(1)); @@ -222,6 +237,11 @@ private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exc .setSettings(Settings.builder().put("index.number_of_shards", 1)) .execute().actionGet(); + final boolean closed = randomBoolean(); + if (closed) { + client().admin().indices().prepareClose("test").get(); + } + ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.getRoutingNodes().unassigned().size(), equalTo(2)); @@ -234,8 +254,11 @@ private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exc assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING)); - healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID) - .setWaitForYellowStatus().execute().actionGet(); + healthResponse = client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForYellowStatus() + .execute().actionGet(); assertThat(healthResponse.isTimedOut(), equalTo(false)); logger.info("--> get the state, verify shard 1 primary allocated"); @@ -244,8 +267,10 @@ private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exc assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.STARTED)); - client().prepareIndex("test", "type", "1").setSource("field", "value") - .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + if (closed == false) { + client().prepareIndex("test", "type", "1").setSource("field", "value") + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + } final Index index = resolveIndex("test"); logger.info("--> closing all nodes"); @@ -263,7 +288,10 @@ private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exc // wait a bit for the cluster to realize that the shard is not there... // TODO can we get around this? the cluster is RED, so what do we wait for? client().admin().cluster().prepareReroute().get(); - assertThat(client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet().getStatus(), + assertThat(client().admin().cluster().prepareHealth() + .setIndices("test") + .setWaitForNodes("2") + .execute().actionGet().getStatus(), equalTo(ClusterHealthStatus.RED)); logger.info("--> explicitly allocate primary"); state = client().admin().cluster().prepareReroute() @@ -294,10 +322,14 @@ public void testRerouteExplain() { assertThat(healthResponse.isTimedOut(), equalTo(false)); logger.info("--> create an index with 1 shard"); - client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) - .execute().actionGet(); + createIndex("test", Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .build()); + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test")); + } ensureGreen("test"); logger.info("--> disable allocation"); @@ -403,12 +435,18 @@ public void testMessageLogging() throws Exception{ Loggers.removeAppender(actionLogger, allocateMockLog); } - public void testClusterRerouteWithBlocks() throws Exception { + public void testClusterRerouteWithBlocks() { List nodesIds = internalCluster().startNodes(2); logger.info("--> create an index with 1 shard and 0 replicas"); - assertAcked(prepareCreate("test-blocks").setSettings(Settings.builder().put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0))); + createIndex("test-blocks", Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .build()); + + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test-blocks")); + } ensureGreen("test-blocks"); logger.info("--> check that the index has 1 shard"); @@ -432,11 +470,14 @@ public void testClusterRerouteWithBlocks() throws Exception { SETTING_READ_ONLY_ALLOW_DELETE)) { try { enableIndexBlock("test-blocks", blockSetting); - assertAcked(client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test-blocks", 0, - nodesIds.get(toggle % 2), nodesIds.get(++toggle % 2)))); + assertAcked(client().admin().cluster().prepareReroute() + .add(new MoveAllocationCommand("test-blocks", 0, nodesIds.get(toggle % 2), nodesIds.get(++toggle % 2)))); - ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForYellowStatus() - .setWaitForNoRelocatingShards(true).execute().actionGet(); + ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth() + .setIndices("test-blocks") + .setWaitForYellowStatus() + .setWaitForNoRelocatingShards(true) + .execute().actionGet(); assertThat(healthResponse.isTimedOut(), equalTo(false)); } finally { disableIndexBlock("test-blocks", blockSetting); diff --git a/server/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java index 6e5af59c2aeab..93bdd674180bf 100644 --- a/server/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java @@ -19,10 +19,9 @@ package org.elasticsearch.cluster.allocation; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -41,14 +40,13 @@ import java.util.List; import java.util.Set; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope= Scope.TEST, numDataNodes =0) public class FilteringAllocationIT extends ESIntegTestCase { - private final Logger logger = LogManager.getLogger(FilteringAllocationIT.class); - - public void testDecommissionNodeNoReplicas() throws Exception { + public void testDecommissionNodeNoReplicas() { logger.info("--> starting 2 nodes"); List nodesIds = internalCluster().startNodes(2); final String node_0 = nodesIds.get(0); @@ -56,10 +54,10 @@ public void testDecommissionNodeNoReplicas() throws Exception { assertThat(cluster().size(), equalTo(2)); logger.info("--> creating an index with no replicas"); - client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put("index.number_of_replicas", 0)) - .execute().actionGet(); - ensureGreen(); + createIndex("test", Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .build()); + ensureGreen("test"); logger.info("--> index some data"); for (int i = 0; i < 100; i++) { client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); @@ -68,11 +66,17 @@ public void testDecommissionNodeNoReplicas() throws Exception { assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet() .getHits().getTotalHits().value, equalTo(100L)); + final boolean closed = randomBoolean(); + if (closed) { + assertAcked(client().admin().indices().prepareClose("test")); + ensureGreen("test"); + } + logger.info("--> decommission the second node"); client().admin().cluster().prepareUpdateSettings() .setTransientSettings(Settings.builder().put("cluster.routing.allocation.exclude._name", node_1)) .execute().actionGet(); - waitForRelocation(); + ensureGreen("test"); logger.info("--> verify all are allocated on node1 now"); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); @@ -84,12 +88,16 @@ public void testDecommissionNodeNoReplicas() throws Exception { } } + if (closed) { + assertAcked(client().admin().indices().prepareOpen("test")); + } + client().admin().indices().prepareRefresh().execute().actionGet(); assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()) .execute().actionGet().getHits().getTotalHits().value, equalTo(100L)); } - public void testDisablingAllocationFiltering() throws Exception { + public void testDisablingAllocationFiltering() { logger.info("--> starting 2 nodes"); List nodesIds = internalCluster().startNodes(2); final String node_0 = nodesIds.get(0); @@ -97,11 +105,11 @@ public void testDisablingAllocationFiltering() throws Exception { assertThat(cluster().size(), equalTo(2)); logger.info("--> creating an index with no replicas"); - client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0)) - .execute().actionGet(); - - ensureGreen(); + createIndex("test", Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .build()); + ensureGreen("test"); logger.info("--> index some data"); for (int i = 0; i < 100; i++) { @@ -110,6 +118,13 @@ public void testDisablingAllocationFiltering() throws Exception { client().admin().indices().prepareRefresh().execute().actionGet(); assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()) .execute().actionGet().getHits().getTotalHits().value, equalTo(100L)); + + final boolean closed = randomBoolean(); + if (closed) { + assertAcked(client().admin().indices().prepareClose("test")); + ensureGreen("test"); + } + ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); IndexRoutingTable indexRoutingTable = clusterState.routingTable().index("test"); int numShardsOnNode1 = 0; @@ -133,7 +148,7 @@ public void testDisablingAllocationFiltering() throws Exception { .setSettings(Settings.builder().put("index.routing.allocation.exclude._name", node_0)) .execute().actionGet(); client().admin().cluster().prepareReroute().get(); - ensureGreen(); + ensureGreen("test"); logger.info("--> verify all shards are allocated on node_1 now"); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); @@ -149,7 +164,7 @@ public void testDisablingAllocationFiltering() throws Exception { .setSettings(Settings.builder().put("index.routing.allocation.exclude._name", "")) .execute().actionGet(); client().admin().cluster().prepareReroute().get(); - ensureGreen(); + ensureGreen("test"); logger.info("--> verify that there are shards allocated on both nodes now"); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); @@ -166,7 +181,7 @@ public void testInvalidIPFilterClusterSettings() { assertEquals("invalid IP address [192.168.1.1.] for [" + filterSetting.getKey() + ipKey + "]", e.getMessage()); } - public void testTransientSettingsStillApplied() throws Exception { + public void testTransientSettingsStillApplied() { List nodes = internalCluster().startNodes(6); Set excludeNodes = new HashSet<>(nodes.subList(0, 3)); Set includeNodes = new HashSet<>(nodes.subList(3, 6)); @@ -177,6 +192,10 @@ public void testTransientSettingsStillApplied() throws Exception { client().admin().indices().prepareCreate("test").get(); ensureGreen("test"); + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test")); + } + Settings exclude = Settings.builder().put("cluster.routing.allocation.exclude._name", Strings.collectionToCommaDelimitedString(excludeNodes)).build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java index f9c0691576f2b..8f6473b0e359d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java @@ -39,13 +39,12 @@ protected int numberOfReplicas() { return 1; } - /** - * Test for - * https://groups.google.com/d/msg/elasticsearch/y-SY_HyoB-8/EZdfNt9VO44J - */ public void testSaneAllocation() { assertAcked(prepareCreate("test", 3)); - ensureGreen(); + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test")); + } + ensureGreen("test"); ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.getRoutingNodes().unassigned().size(), equalTo(0)); @@ -56,7 +55,7 @@ public void testSaneAllocation() { } client().admin().indices().prepareUpdateSettings("test") .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 0)).execute().actionGet(); - ensureGreen(); + ensureGreen("test"); state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.getRoutingNodes().unassigned().size(), equalTo(0)); @@ -68,11 +67,14 @@ public void testSaneAllocation() { // create another index assertAcked(prepareCreate("test2", 3)); - ensureGreen(); + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test2")); + } + ensureGreen("test2"); client().admin().indices().prepareUpdateSettings("test") .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1)).execute().actionGet(); - ensureGreen(); + ensureGreen("test"); state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.getRoutingNodes().unassigned().size(), equalTo(0)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 5c96f6f1cbcd4..571843126f98c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -155,8 +155,7 @@ public void testIndexOptionsLenient() { .put(indexBuilder("foofoo").putAlias(AliasMetaData.builder("barbaz"))); ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); - IndicesOptions lenientExpand = IndicesOptions.fromOptions(true, true, true, true); - IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.lenientExpandOpen(), lenientExpand}; + IndicesOptions[] indicesOptions = new IndicesOptions[]{IndicesOptions.lenientExpandOpen(), IndicesOptions.lenientExpand()}; for (IndicesOptions options : indicesOptions) { IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options); String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo"); @@ -199,7 +198,7 @@ public void testIndexOptionsLenient() { String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY); assertEquals(3, results.length); - context = new IndexNameExpressionResolver.Context(state, lenientExpand); + context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpand()); results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY); assertEquals(Arrays.toString(results), 4, results.length); @@ -208,7 +207,7 @@ public void testIndexOptionsLenient() { assertEquals(3, results.length); assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo")); - context = new IndexNameExpressionResolver.Context(state, lenientExpand); + context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpand()); results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo*"); assertEquals(4, results.length); assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo", "foofoo-closed")); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java index 56ee25ee5febb..6ba85cd22a36e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.shards.ClusterShardLimitIT; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ValidationException; @@ -114,6 +115,60 @@ public void testCloseRoutingTable() { } } + public void testCloseRoutingTableRemovesRoutingTable() { + final Set nonBlockedIndices = new HashSet<>(); + final Map blockedIndices = new HashMap<>(); + final Map results = new HashMap<>(); + final ClusterBlock closingBlock = MetaDataIndexStateService.createIndexClosingBlock(); + + ClusterState state = ClusterState.builder(new ClusterName("testCloseRoutingTableRemovesRoutingTable")).build(); + for (int i = 0; i < randomIntBetween(1, 25); i++) { + final String indexName = "index-" + i; + + if (randomBoolean()) { + state = addOpenedIndex(indexName, randomIntBetween(1, 5), randomIntBetween(0, 5), state); + nonBlockedIndices.add(state.metaData().index(indexName).getIndex()); + } else { + state = addBlockedIndex(indexName, randomIntBetween(1, 5), randomIntBetween(0, 5), state, closingBlock); + blockedIndices.put(state.metaData().index(indexName).getIndex(), closingBlock); + results.put(state.metaData().index(indexName).getIndex(), new AcknowledgedResponse(randomBoolean())); + } + } + + state = ClusterState.builder(state) + .nodes(DiscoveryNodes.builder(state.nodes()) + .add(new DiscoveryNode("old_node", buildNewFakeTransportAddress(), emptyMap(), + new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.V_7_1_0)) + .add(new DiscoveryNode("new_node", buildNewFakeTransportAddress(), emptyMap(), + new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.V_8_0_0))) + .build(); + + state = MetaDataIndexStateService.closeRoutingTable(state, blockedIndices, results); + assertThat(state.metaData().indices().size(), equalTo(nonBlockedIndices.size() + blockedIndices.size())); + + for (Index nonBlockedIndex : nonBlockedIndices) { + assertIsOpened(nonBlockedIndex.getName(), state); + assertThat(state.blocks().hasIndexBlockWithId(nonBlockedIndex.getName(), INDEX_CLOSED_BLOCK_ID), is(false)); + } + for (Index blockedIndex : blockedIndices.keySet()) { + if (results.get(blockedIndex).isAcknowledged()) { + IndexMetaData indexMetaData = state.metaData().index(blockedIndex); + assertThat(indexMetaData.getState(), is(IndexMetaData.State.CLOSE)); + Settings indexSettings = indexMetaData.getSettings(); + assertThat(indexSettings.hasValue(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey()), is(false)); + assertThat(state.blocks().hasIndexBlock(blockedIndex.getName(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK), is(true)); + assertThat("Index must have only 1 block with [id=" + MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID + "]", + state.blocks().indices().getOrDefault(blockedIndex.getName(), emptySet()).stream() + .filter(clusterBlock -> clusterBlock.id() == MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID).count(), equalTo(1L)); + assertThat("Index routing table should have been removed when closing the index on mixed cluster version", + state.routingTable().index(blockedIndex), nullValue()); + } else { + assertIsOpened(blockedIndex.getName(), state); + assertThat(state.blocks().hasIndexBlock(blockedIndex.getName(), closingBlock), is(true)); + } + } + } + public void testAddIndexClosedBlocks() { final ClusterState initialState = ClusterState.builder(new ClusterName("testAddIndexClosedBlocks")).build(); { @@ -139,7 +194,6 @@ public void testAddIndexClosedBlocks() { ClusterState updatedState = MetaDataIndexStateService.addIndexClosedBlocks(indices, blockedIndices, state); assertSame(state, updatedState); assertTrue(blockedIndices.isEmpty()); - } { final Map blockedIndices = new HashMap<>(); @@ -190,14 +244,6 @@ public void testAddIndexClosedBlocks() { ClusterState state = addOpenedIndex("index-1", randomIntBetween(1, 3), randomIntBetween(0, 3), initialState); state = addOpenedIndex("index-2", randomIntBetween(1, 3), randomIntBetween(0, 3), state); state = addOpenedIndex("index-3", randomIntBetween(1, 3), randomIntBetween(0, 3), state); - final boolean mixedVersions = randomBoolean(); - if (mixedVersions) { - state = ClusterState.builder(state) - .nodes(DiscoveryNodes.builder(state.nodes()) - .add(new DiscoveryNode("old_node", buildNewFakeTransportAddress(), emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.V_6_0_0))) - .build(); - } Index index1 = state.metaData().index("index-1").getIndex(); Index index2 = state.metaData().index("index-2").getIndex(); @@ -209,11 +255,7 @@ public void testAddIndexClosedBlocks() { for (Index index : indices) { assertTrue(blockedIndices.containsKey(index)); - if (mixedVersions) { - assertIsClosed(index.getName(), updatedState); - } else { - assertHasBlock(index.getName(), updatedState, blockedIndices.get(index)); - } + assertHasBlock(index.getName(), updatedState, blockedIndices.get(index)); } } } @@ -262,6 +304,32 @@ public void testValidateShardLimit() { currentShards + "]/[" + maxShards + "] maximum shards open;", exception.getMessage()); } + public void testIsIndexVerifiedBeforeClosed() { + final ClusterState initialState = ClusterState.builder(new ClusterName("testIsIndexMetaDataClosed")).build(); + { + String indexName = "open"; + ClusterState state = addOpenedIndex(indexName, randomIntBetween(1, 3), randomIntBetween(0, 3), initialState); + assertFalse(MetaDataIndexStateService.isIndexVerifiedBeforeClosed(state.getMetaData().index(indexName))); + } + { + String indexName = "closed"; + ClusterState state = addClosedIndex(indexName, randomIntBetween(1, 3), randomIntBetween(0, 3), initialState); + assertTrue(MetaDataIndexStateService.isIndexVerifiedBeforeClosed(state.getMetaData().index(indexName))); + } + { + String indexName = "closed-no-setting"; + IndexMetaData indexMetaData = IndexMetaData.builder(indexName) + .state(IndexMetaData.State.CLOSE) + .creationDate(randomNonNegativeLong()) + .settings(Settings.builder() + .put(SETTING_VERSION_CREATED, Version.CURRENT) + .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3)) + .put(SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 3))) + .build(); + assertFalse(MetaDataIndexStateService.isIndexVerifiedBeforeClosed(indexMetaData)); + } + } + public static ClusterState createClusterForShardLimitTest(int nodesInCluster, int openIndexShards, int openIndexReplicas, int closedIndexShards, int closedIndexReplicas, Settings clusterSettings) { ImmutableOpenMap.Builder dataNodes = ImmutableOpenMap.builder(); @@ -334,31 +402,35 @@ private static ClusterState addIndex(final ClusterState currentState, final int numReplicas, final IndexMetaData.State state, @Nullable final ClusterBlock block) { + + final Settings.Builder settings = Settings.builder() + .put(SETTING_VERSION_CREATED, Version.CURRENT) + .put(SETTING_NUMBER_OF_SHARDS, numShards) + .put(SETTING_NUMBER_OF_REPLICAS, numReplicas); + if (state == IndexMetaData.State.CLOSE) { + settings.put(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey(), true); + } final IndexMetaData indexMetaData = IndexMetaData.builder(index) .state(state) .creationDate(randomNonNegativeLong()) - .settings(Settings.builder() - .put(SETTING_VERSION_CREATED, Version.CURRENT) - .put(SETTING_NUMBER_OF_SHARDS, numShards) - .put(SETTING_NUMBER_OF_REPLICAS, numReplicas)) + .settings(settings) .build(); final ClusterState.Builder clusterStateBuilder = ClusterState.builder(currentState); clusterStateBuilder.metaData(MetaData.builder(currentState.metaData()).put(indexMetaData, true)); - if (state == IndexMetaData.State.OPEN) { - final IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetaData.getIndex()); - for (int j = 0; j < indexMetaData.getNumberOfShards(); j++) { - ShardId shardId = new ShardId(indexMetaData.getIndex(), j); - IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - indexShardRoutingBuilder.addShard(newShardRouting(shardId, randomAlphaOfLength(10), true, ShardRoutingState.STARTED)); - for (int k = 0; k < indexMetaData.getNumberOfReplicas(); k++) { - indexShardRoutingBuilder.addShard(newShardRouting(shardId, randomAlphaOfLength(10), false, ShardRoutingState.STARTED)); - } - indexRoutingTable.addIndexShard(indexShardRoutingBuilder.build()); + final IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetaData.getIndex()); + for (int j = 0; j < indexMetaData.getNumberOfShards(); j++) { + ShardId shardId = new ShardId(indexMetaData.getIndex(), j); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + indexShardRoutingBuilder.addShard(newShardRouting(shardId, randomAlphaOfLength(10), true, ShardRoutingState.STARTED)); + for (int k = 0; k < indexMetaData.getNumberOfReplicas(); k++) { + indexShardRoutingBuilder.addShard(newShardRouting(shardId, randomAlphaOfLength(10), false, ShardRoutingState.STARTED)); } - clusterStateBuilder.routingTable(RoutingTable.builder(currentState.routingTable()).add(indexRoutingTable).build()); + indexRoutingTable.addIndexShard(indexShardRoutingBuilder.build()); } + clusterStateBuilder.routingTable(RoutingTable.builder(currentState.routingTable()).add(indexRoutingTable).build()); + if (block != null) { clusterStateBuilder.blocks(ClusterBlocks.builder().blocks(currentState.blocks()).addIndexBlock(index, block)); } @@ -366,17 +438,33 @@ private static ClusterState addIndex(final ClusterState currentState, } private static void assertIsOpened(final String indexName, final ClusterState clusterState) { - assertThat(clusterState.metaData().index(indexName).getState(), is(IndexMetaData.State.OPEN)); + final IndexMetaData indexMetaData = clusterState.metaData().indices().get(indexName); + assertThat(indexMetaData.getState(), is(IndexMetaData.State.OPEN)); + assertThat(indexMetaData.getSettings().hasValue(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey()), is(false)); + assertThat(clusterState.routingTable().index(indexName), notNullValue()); + assertThat(clusterState.blocks().hasIndexBlock(indexName, MetaDataIndexStateService.INDEX_CLOSED_BLOCK), is(false)); assertThat(clusterState.routingTable().index(indexName), notNullValue()); } private static void assertIsClosed(final String indexName, final ClusterState clusterState) { - assertThat(clusterState.metaData().index(indexName).getState(), is(IndexMetaData.State.CLOSE)); - assertThat(clusterState.routingTable().index(indexName), nullValue()); + final IndexMetaData indexMetaData = clusterState.metaData().indices().get(indexName); + assertThat(indexMetaData.getState(), is(IndexMetaData.State.CLOSE)); + final Settings indexSettings = indexMetaData.getSettings(); + assertThat(indexSettings.hasValue(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey()), is(true)); + assertThat(indexSettings.getAsBoolean(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey(), false), is(true)); assertThat(clusterState.blocks().hasIndexBlock(indexName, MetaDataIndexStateService.INDEX_CLOSED_BLOCK), is(true)); assertThat("Index " + indexName + " must have only 1 block with [id=" + MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID + "]", clusterState.blocks().indices().getOrDefault(indexName, emptySet()).stream() .filter(clusterBlock -> clusterBlock.id() == MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID).count(), equalTo(1L)); + + final IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(indexName); + assertThat(indexRoutingTable, notNullValue()); + + for(IndexShardRoutingTable shardRoutingTable : indexRoutingTable) { + assertThat(shardRoutingTable.shards().stream().allMatch(ShardRouting::unassigned), is(true)); + assertThat(shardRoutingTable.shards().stream().map(ShardRouting::unassignedInfo).map(UnassignedInfo::getReason) + .allMatch(info -> info == UnassignedInfo.Reason.INDEX_CLOSED), is(true)); + } } private static void assertHasBlock(final String indexName, final ClusterState clusterState, final ClusterBlock closingBlock) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index a6c2fab5c91e4..851fe9c550270 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.node.DiscoveryNodes.Builder; import org.elasticsearch.cluster.routing.allocation.AllocationService; @@ -38,6 +39,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -373,6 +375,36 @@ public void testDistinctNodes() { assertFalse(IndexShardRoutingTable.Builder.distinctNodes(Arrays.asList(routing2, routing4))); } + public void testAddAsRecovery() { + { + final IndexMetaData indexMetaData = createIndexMetaData(TEST_INDEX_1).state(IndexMetaData.State.OPEN).build(); + final RoutingTable routingTable = new RoutingTable.Builder().addAsRecovery(indexMetaData).build(); + assertThat(routingTable.hasIndex(TEST_INDEX_1), is(true)); + assertThat(routingTable.allShards(TEST_INDEX_1).size(), is(this.shardsPerIndex)); + assertThat(routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size(), is(this.shardsPerIndex)); + } + { + final IndexMetaData indexMetaData = createIndexMetaData(TEST_INDEX_1).state(IndexMetaData.State.CLOSE).build(); + final RoutingTable routingTable = new RoutingTable.Builder().addAsRecovery(indexMetaData).build(); + assertThat(routingTable.hasIndex(TEST_INDEX_1), is(false)); + expectThrows(IndexNotFoundException.class, () -> routingTable.allShards(TEST_INDEX_1)); + } + { + final IndexMetaData indexMetaData = createIndexMetaData(TEST_INDEX_1).build(); + final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(indexMetaData) + .state(IndexMetaData.State.CLOSE) + .settings(Settings.builder() + .put(indexMetaData.getSettings()) + .put(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey(), true) + .build()) + .settingsVersion(indexMetaData.getSettingsVersion() + 1); + final RoutingTable routingTable = new RoutingTable.Builder().addAsRecovery(indexMetaDataBuilder.build()).build(); + assertThat(routingTable.hasIndex(TEST_INDEX_1), is(true)); + assertThat(routingTable.allShards(TEST_INDEX_1).size(), is(this.shardsPerIndex)); + assertThat(routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size(), is(this.shardsPerIndex)); + } + } + /** reverse engineer the in sync aid based on the given indexRoutingTable **/ public static IndexMetaData updateActiveAllocations(IndexRoutingTable indexRoutingTable, IndexMetaData indexMetaData) { IndexMetaData.Builder imdBuilder = IndexMetaData.builder(indexMetaData); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java index 16cde1a990907..bc3191c14dfba 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.nio.ByteBuffer; @@ -54,6 +55,7 @@ import static org.hamcrest.Matchers.nullValue; public class UnassignedInfoTests extends ESAllocationTestCase { + public void testReasonOrdinalOrder() { UnassignedInfo.Reason[] order = new UnassignedInfo.Reason[]{ UnassignedInfo.Reason.INDEX_CREATED, @@ -70,7 +72,8 @@ public void testReasonOrdinalOrder() { UnassignedInfo.Reason.REALLOCATED_REPLICA, UnassignedInfo.Reason.PRIMARY_FAILED, UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY, - UnassignedInfo.Reason.MANUAL_ALLOCATION,}; + UnassignedInfo.Reason.MANUAL_ALLOCATION, + UnassignedInfo.Reason.INDEX_CLOSED,}; for (int i = 0; i < order.length; i++) { assertThat(order[i].ordinal(), equalTo(i)); } @@ -95,6 +98,21 @@ public void testSerialization() throws Exception { assertThat(read.getNumFailedAllocations(), equalTo(meta.getNumFailedAllocations())); } + public void testBwcSerialization() throws Exception { + final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CLOSED, "message"); + BytesStreamOutput out = new BytesStreamOutput(); + out.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0))); + unassignedInfo.writeTo(out); + out.close(); + + UnassignedInfo read = new UnassignedInfo(out.bytes().streamInput()); + assertThat(read.getReason(), equalTo(UnassignedInfo.Reason.REINITIALIZED)); + assertThat(read.getUnassignedTimeInMillis(), equalTo(unassignedInfo.getUnassignedTimeInMillis())); + assertThat(read.getMessage(), equalTo(unassignedInfo.getMessage())); + assertThat(read.getDetails(), equalTo(unassignedInfo.getDetails())); + assertThat(read.getNumFailedAllocations(), equalTo(unassignedInfo.getNumFailedAllocations())); + } + public void testIndexCreated() { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)) diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index cae33db90a6bc..eec02438f0031 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.coordination.CoordinationMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; @@ -241,11 +242,36 @@ public void testUpdateRoutingTable() { .build(); assertFalse(initialState.routingTable().hasIndex(index)); - final ClusterState newState = updateRoutingTable(initialState); - - assertTrue(newState.routingTable().hasIndex(index)); - assertThat(newState.routingTable().version(), is(0L)); - assertThat(newState.routingTable().allShards(index.getName()).size(), is(numOfShards)); + { + final ClusterState newState = updateRoutingTable(initialState); + assertTrue(newState.routingTable().hasIndex(index)); + assertThat(newState.routingTable().version(), is(0L)); + assertThat(newState.routingTable().allShards(index.getName()).size(), is(numOfShards)); + } + { + final ClusterState newState = updateRoutingTable(ClusterState.builder(initialState) + .metaData(MetaData.builder(initialState.metaData()) + .put(IndexMetaData.builder(initialState.metaData().index("test")) + .state(IndexMetaData.State.CLOSE)) + .build()) + .build()); + assertFalse(newState.routingTable().hasIndex(index)); + } + { + final ClusterState newState = updateRoutingTable(ClusterState.builder(initialState) + .metaData(MetaData.builder(initialState.metaData()) + .put(IndexMetaData.builder(initialState.metaData().index("test")) + .state(IndexMetaData.State.CLOSE) + .settings(Settings.builder() + .put(initialState.metaData().index("test").getSettings()) + .put(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey(), true) + .build()) + ).build()) + .build()); + assertTrue(newState.routingTable().hasIndex(index)); + assertThat(newState.routingTable().version(), is(0L)); + assertThat(newState.routingTable().allShards(index.getName()).size(), is(numOfShards)); + } } public void testMixCurrentAndRecoveredState() { diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index ebdae985a39c7..5b30b85cb72ec 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -54,10 +54,11 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.notNullValue; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class GatewayIndexStateIT extends ESIntegTestCase { @@ -113,11 +114,11 @@ public void testSimpleOpenClose() throws Exception { client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get(); logger.info("--> closing test index..."); - client().admin().indices().prepareClose("test").get(); + assertAcked(client().admin().indices().prepareClose("test")); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); - assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); + assertThat(stateResponse.getState().routingTable().index("test"), notNullValue()); logger.info("--> verifying that the state is green"); ensureGreen(); @@ -136,7 +137,7 @@ public void testSimpleOpenClose() throws Exception { ensureGreen(); logger.info("--> opening the first index again..."); - client().admin().indices().prepareOpen("test").execute().actionGet(); + assertAcked(client().admin().indices().prepareOpen("test")); logger.info("--> verifying that the state is green"); ensureGreen(); @@ -152,10 +153,10 @@ public void testSimpleOpenClose() throws Exception { assertThat(getResponse.isExists(), equalTo(true)); logger.info("--> closing test index..."); - client().admin().indices().prepareClose("test").execute().actionGet(); + assertAcked(client().admin().indices().prepareClose("test")); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); - assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); + assertThat(stateResponse.getState().routingTable().index("test"), notNullValue()); logger.info("--> restarting nodes..."); internalCluster().fullRestart(); @@ -164,7 +165,7 @@ public void testSimpleOpenClose() throws Exception { stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); - assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); + assertThat(stateResponse.getState().routingTable().index("test"), notNullValue()); logger.info("--> trying to index into a closed index ..."); try { @@ -252,11 +253,11 @@ public void testTwoNodesSingleDoc() throws Exception { } logger.info("--> closing test index..."); - client().admin().indices().prepareClose("test").execute().actionGet(); + assertAcked(client().admin().indices().prepareClose("test")); ClusterStateResponse stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); - assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); + assertThat(stateResponse.getState().routingTable().index("test"), notNullValue()); logger.info("--> opening the index..."); client().admin().indices().prepareOpen("test").execute().actionGet(); diff --git a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java index a47d4db2a2579..e5e554818c020 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -47,6 +48,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.test.InternalSettingsPlugin.TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.core.IsEqual.equalTo; /** Unit test(s) for IndexService */ @@ -109,7 +111,6 @@ protected String getThreadPool() { latch2.get().countDown(); assertEquals(2, count.get()); - task = new IndexService.BaseAsyncTask(indexService, TimeValue.timeValueMillis(1000000)) { @Override protected void runInternal() { @@ -117,6 +118,34 @@ protected void runInternal() { } }; assertTrue(task.mustReschedule()); + + // now close the index + final Index index = indexService.index(); + assertAcked(client().admin().indices().prepareClose(index.getName())); + awaitBusy(() -> getInstanceFromNode(IndicesService.class).hasIndex(index)); + + final IndexService closedIndexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index); + assertNotSame(indexService, closedIndexService); + assertFalse(task.mustReschedule()); + assertFalse(task.isClosed()); + assertEquals(1000000, task.getInterval().millis()); + + // now reopen the index + assertAcked(client().admin().indices().prepareOpen(index.getName())); + awaitBusy(() -> getInstanceFromNode(IndicesService.class).hasIndex(index)); + indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index); + assertNotSame(closedIndexService, indexService); + + task = new IndexService.BaseAsyncTask(indexService, TimeValue.timeValueMillis(100000)) { + @Override + protected void runInternal() { + + } + }; + assertTrue(task.mustReschedule()); + assertFalse(task.isClosed()); + assertTrue(task.isScheduled()); + indexService.close("simon says", false); assertFalse("no shards left", task.mustReschedule()); assertTrue(task.isScheduled()); @@ -124,7 +153,7 @@ protected void runInternal() { assertFalse(task.isScheduled()); } - public void testRefreshTaskIsUpdated() throws IOException { + public void testRefreshTaskIsUpdated() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); IndexService.AsyncRefreshTask refreshTask = indexService.getRefreshTask(); assertEquals(1000, refreshTask.getInterval().millis()); @@ -167,12 +196,35 @@ public void testRefreshTaskIsUpdated() throws IOException { assertTrue(refreshTask.isScheduled()); assertFalse(refreshTask.isClosed()); assertEquals(200, refreshTask.getInterval().millis()); + + // now close the index + final Index index = indexService.index(); + assertAcked(client().admin().indices().prepareClose(index.getName())); + awaitBusy(() -> getInstanceFromNode(IndicesService.class).hasIndex(index)); + + final IndexService closedIndexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index); + assertNotSame(indexService, closedIndexService); + assertNotSame(refreshTask, closedIndexService.getRefreshTask()); + assertFalse(closedIndexService.getRefreshTask().mustReschedule()); + assertFalse(closedIndexService.getRefreshTask().isClosed()); + assertEquals(200, closedIndexService.getRefreshTask().getInterval().millis()); + + // now reopen the index + assertAcked(client().admin().indices().prepareOpen(index.getName())); + awaitBusy(() -> getInstanceFromNode(IndicesService.class).hasIndex(index)); + indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index); + assertNotSame(closedIndexService, indexService); + refreshTask = indexService.getRefreshTask(); + assertTrue(indexService.getRefreshTask().mustReschedule()); + assertTrue(refreshTask.isScheduled()); + assertFalse(refreshTask.isClosed()); + indexService.close("simon says", false); assertFalse(refreshTask.isScheduled()); assertTrue(refreshTask.isClosed()); } - public void testFsyncTaskIsRunning() throws IOException { + public void testFsyncTaskIsRunning() throws Exception { Settings settings = Settings.builder() .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC).build(); IndexService indexService = createIndex("test", settings); @@ -182,6 +234,28 @@ public void testFsyncTaskIsRunning() throws IOException { assertTrue(fsyncTask.mustReschedule()); assertTrue(fsyncTask.isScheduled()); + // now close the index + final Index index = indexService.index(); + assertAcked(client().admin().indices().prepareClose(index.getName())); + awaitBusy(() -> getInstanceFromNode(IndicesService.class).hasIndex(index)); + + final IndexService closedIndexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index); + assertNotSame(indexService, closedIndexService); + assertNotSame(fsyncTask, closedIndexService.getFsyncTask()); + assertFalse(closedIndexService.getFsyncTask().mustReschedule()); + assertFalse(closedIndexService.getFsyncTask().isClosed()); + assertEquals(5000, closedIndexService.getFsyncTask().getInterval().millis()); + + // now reopen the index + assertAcked(client().admin().indices().prepareOpen(index.getName())); + awaitBusy(() -> getInstanceFromNode(IndicesService.class).hasIndex(index)); + indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index); + assertNotSame(closedIndexService, indexService); + fsyncTask = indexService.getFsyncTask(); + assertTrue(indexService.getRefreshTask().mustReschedule()); + assertTrue(fsyncTask.isScheduled()); + assertFalse(fsyncTask.isClosed()); + indexService.close("simon says", false); assertFalse(fsyncTask.isScheduled()); assertTrue(fsyncTask.isClosed()); diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineRecoveryTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineRecoveryTests.java new file mode 100644 index 0000000000000..7e8f18dd005fc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineRecoveryTests.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.engine; + +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardTestCase; + +import java.io.IOException; + +import static org.elasticsearch.cluster.routing.ShardRoutingHelper.initWithSameId; + +public class NoOpEngineRecoveryTests extends IndexShardTestCase { + + public void testRecoverFromNoOp() throws IOException { + final int nbDocs = scaledRandomIntBetween(1, 100); + + final IndexShard indexShard = newStartedShard(true); + for (int i = 0; i < nbDocs; i++) { + indexDoc(indexShard, "_doc", String.valueOf(i)); + } + indexShard.close("test", true); + + final ShardRouting shardRouting = indexShard.routingEntry(); + IndexShard primary = reinitShard(indexShard, initWithSameId(shardRouting, ExistingStoreRecoverySource.INSTANCE), NoOpEngine::new); + recoverShardFromStore(primary); + assertEquals(primary.seqNoStats().getMaxSeqNo(), primary.getMaxSeqNoOfUpdatesOrDeletes()); + assertEquals(nbDocs, primary.docStats().getCount()); + + IndexShard replica = newShard(false, Settings.EMPTY, NoOpEngine::new); + recoverReplica(replica, primary, true); + assertEquals(replica.seqNoStats().getMaxSeqNo(), replica.getMaxSeqNoOfUpdatesOrDeletes()); + assertEquals(nbDocs, replica.docStats().getCount()); + closeShards(primary, replica); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java new file mode 100644 index 0000000000000..b70ccf03aacaf --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java @@ -0,0 +1,158 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.engine; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.store.LockObtainFailedException; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.seqno.ReplicationTracker; +import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.translog.TranslogDeletionPolicy; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicLong; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class NoOpEngineTests extends EngineTestCase { + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY); + + public void testNoopEngine() throws IOException { + engine.close(); + final NoOpEngine engine = new NoOpEngine(noOpConfig(INDEX_SETTINGS, store, primaryTranslogDir)); + expectThrows(UnsupportedOperationException.class, () -> engine.syncFlush(null, null)); + assertThat(engine.refreshNeeded(), equalTo(false)); + assertThat(engine.shouldPeriodicallyFlush(), equalTo(false)); + engine.close(); + } + + public void testTwoNoopEngines() throws IOException { + engine.close(); + // Ensure that we can't open two noop engines for the same store + final EngineConfig engineConfig = noOpConfig(INDEX_SETTINGS, store, primaryTranslogDir); + try (NoOpEngine ignored = new NoOpEngine(engineConfig)) { + UncheckedIOException e = expectThrows(UncheckedIOException.class, () -> new NoOpEngine(engineConfig)); + assertThat(e.getCause(), instanceOf(LockObtainFailedException.class)); + } + } + + public void testNoopAfterRegularEngine() throws IOException { + int docs = randomIntBetween(1, 10); + ReplicationTracker tracker = (ReplicationTracker) engine.config().getGlobalCheckpointSupplier(); + ShardRouting routing = TestShardRouting.newShardRouting("test", shardId.id(), "node", + null, true, ShardRoutingState.STARTED, allocationId); + IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(shardId).addShard(routing).build(); + tracker.updateFromMaster(1L, Collections.singleton(allocationId.getId()), table, Collections.emptySet()); + tracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED); + for (int i = 0; i < docs; i++) { + ParsedDocument doc = testParsedDocument("" + i, null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); + tracker.updateLocalCheckpoint(allocationId.getId(), i); + } + + flushAndTrimTranslog(engine); + + long localCheckpoint = engine.getLocalCheckpoint(); + long maxSeqNo = engine.getSeqNoStats(100L).getMaxSeqNo(); + engine.close(); + + final NoOpEngine noOpEngine = new NoOpEngine(noOpConfig(INDEX_SETTINGS, store, primaryTranslogDir, tracker)); + assertThat(noOpEngine.getLocalCheckpoint(), equalTo(localCheckpoint)); + assertThat(noOpEngine.getSeqNoStats(100L).getMaxSeqNo(), equalTo(maxSeqNo)); + try (Engine.IndexCommitRef ref = noOpEngine.acquireLastIndexCommit(false)) { + try (IndexReader reader = DirectoryReader.open(ref.getIndexCommit())) { + assertThat(reader.numDocs(), equalTo(docs)); + } + } + noOpEngine.close(); + } + + public void testNoOpEngineDocStats() throws Exception { + IOUtils.close(engine, store); + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + try (Store store = createStore()) { + Path translogPath = createTempDir(); + EngineConfig config = config(defaultSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get); + final int numDocs = scaledRandomIntBetween(10, 3000); + int deletions = 0; + try (InternalEngine engine = createEngine(config)) { + for (int i = 0; i < numDocs; i++) { + engine.index(indexForDoc(createParsedDoc(Integer.toString(i), null))); + if (rarely()) { + engine.flush(); + } + globalCheckpoint.set(engine.getLocalCheckpoint()); + } + + for (int i = 0; i < numDocs; i++) { + if (randomBoolean()) { + String delId = Integer.toString(i); + Engine.DeleteResult result = engine.delete(new Engine.Delete("test", delId, newUid(delId), primaryTerm.get())); + assertTrue(result.isFound()); + globalCheckpoint.set(engine.getLocalCheckpoint()); + deletions += 1; + } + } + engine.getLocalCheckpointTracker().waitForOpsToComplete(numDocs + deletions - 1); + flushAndTrimTranslog(engine); + } + + final DocsStats expectedDocStats; + try (InternalEngine engine = createEngine(config)) { + expectedDocStats = engine.docStats(); + } + + try (NoOpEngine noOpEngine = new NoOpEngine(config)) { + assertEquals(expectedDocStats.getCount(), noOpEngine.docStats().getCount()); + assertEquals(expectedDocStats.getDeleted(), noOpEngine.docStats().getDeleted()); + assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes()); + assertEquals(expectedDocStats.getAverageSizeInBytes(), noOpEngine.docStats().getAverageSizeInBytes()); + } catch (AssertionError e) { + logger.error(config.getMergePolicy()); + throw e; + } + } + } + + private void flushAndTrimTranslog(final InternalEngine engine) { + engine.flush(true, true); + final TranslogDeletionPolicy deletionPolicy = engine.getTranslog().getDeletionPolicy(); + deletionPolicy.setRetentionSizeInBytes(-1); + deletionPolicy.setRetentionAgeInMillis(-1); + deletionPolicy.setMinTranslogGenerationForRecovery(engine.getTranslog().getGeneration().translogFileGeneration); + engine.flush(true, true); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 225637567972c..78395cf544ad3 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -38,7 +38,6 @@ import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedRunnable; @@ -62,6 +61,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.NoOpEngine; import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.mapper.SourceToParse; @@ -87,7 +87,6 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -103,14 +102,16 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; +import java.util.stream.Stream; +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLength; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting; import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.elasticsearch.index.shard.IndexShardTestCase.getTranslog; @@ -279,73 +280,63 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException { } public void testIndexCanChangeCustomDataPath() throws Exception { - Environment env = getInstanceFromNode(Environment.class); - Path idxPath = env.sharedDataFile().resolve(randomAlphaOfLength(10)); - final String INDEX = "idx"; - Path startDir = idxPath.resolve("start-" + randomAlphaOfLength(10)); - Path endDir = idxPath.resolve("end-" + randomAlphaOfLength(10)); - logger.info("--> start dir: [{}]", startDir.toAbsolutePath().toString()); - logger.info("--> end dir: [{}]", endDir.toAbsolutePath().toString()); - // temp dirs are automatically created, but the end dir is what - // startDir is going to be renamed as, so it needs to be deleted - // otherwise we get all sorts of errors about the directory - // already existing - IOUtils.rm(endDir); - - Settings sb = Settings.builder() - .put(IndexMetaData.SETTING_DATA_PATH, startDir.toAbsolutePath().toString()) - .build(); - Settings sb2 = Settings.builder() - .put(IndexMetaData.SETTING_DATA_PATH, endDir.toAbsolutePath().toString()) - .build(); + final String index = "test-custom-data-path"; + final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataFile().resolve(randomAsciiLettersOfLength(10)); + final Path indexDataPath = sharedDataPath.resolve("start-" + randomAsciiLettersOfLength(10)); - logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString()); - createIndex(INDEX, sb); - ensureGreen(INDEX); - client().prepareIndex(INDEX, "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get(); + logger.info("--> creating index [{}] with data_path [{}]", index, indexDataPath); + createIndex(index, Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, indexDataPath.toAbsolutePath().toString()).build()); + client().prepareIndex(index, "bar", "1").setSource("foo", "bar").setRefreshPolicy(IMMEDIATE).get(); + ensureGreen(index); - SearchResponse resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get(); - assertThat("found the hit", resp.getHits().getTotalHits().value, equalTo(1L)); + assertHitCount(client().prepareSearch(index).setSize(0).get(), 1L); - logger.info("--> closing the index [{}]", INDEX); - client().admin().indices().prepareClose(INDEX).get(); + logger.info("--> closing the index [{}]", index); + assertAcked(client().admin().indices().prepareClose(index)); logger.info("--> index closed, re-opening..."); - client().admin().indices().prepareOpen(INDEX).get(); + assertAcked(client().admin().indices().prepareOpen(index)); logger.info("--> index re-opened"); - ensureGreen(INDEX); + ensureGreen(index); - resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get(); - assertThat("found the hit", resp.getHits().getTotalHits().value, equalTo(1L)); + assertHitCount(client().prepareSearch(index).setSize(0).get(), 1L); // Now, try closing and changing the settings + logger.info("--> closing the index [{}] before updating data_path", index); + assertAcked(client().admin().indices().prepareClose(index)); - logger.info("--> closing the index [{}]", INDEX); - client().admin().indices().prepareClose(INDEX).get(); - - logger.info("--> moving data on disk [{}] to [{}]", startDir.getFileName(), endDir.getFileName()); - assert Files.exists(endDir) == false : "end directory should not exist!"; - Files.move(startDir, endDir, StandardCopyOption.REPLACE_EXISTING); + final Path newIndexDataPath = sharedDataPath.resolve("end-" + randomAlphaOfLength(10)); + IOUtils.rm(newIndexDataPath); - logger.info("--> updating settings..."); - client().admin().indices().prepareUpdateSettings(INDEX) - .setSettings(sb2) - .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true)) - .get(); + logger.info("--> copying data on disk from [{}] to [{}]", indexDataPath, newIndexDataPath); + assert Files.exists(newIndexDataPath) == false : "new index data path directory should not exist!"; + try (Stream stream = Files.walk(indexDataPath)) { + stream.forEach(path -> { + try { + if (path.endsWith(".lock") == false) { + Files.copy(path, newIndexDataPath.resolve(indexDataPath.relativize(path))); + } + } catch (final Exception e) { + logger.error("Failed to copy data path directory", e); + fail(); + } + }); + } - assert Files.exists(startDir) == false : "start dir shouldn't exist"; + logger.info("--> updating data_path to [{}] for index [{}]", newIndexDataPath, index); + assertAcked(client().admin().indices().prepareUpdateSettings(index) + .setSettings(Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, newIndexDataPath.toAbsolutePath().toString()).build()) + .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true))); logger.info("--> settings updated and files moved, re-opening index"); - client().admin().indices().prepareOpen(INDEX).get(); + assertAcked(client().admin().indices().prepareOpen(index)); logger.info("--> index re-opened"); - ensureGreen(INDEX); + ensureGreen(index); - resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get(); - assertThat("found the hit", resp.getHits().getTotalHits().value, equalTo(1L)); + assertHitCount(client().prepareSearch(index).setSize(0).get(), 1L); - assertAcked(client().admin().indices().prepareDelete(INDEX)); + assertAcked(client().admin().indices().prepareDelete(index)); assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class))); - assertPathHasBeenCleared(startDir.toAbsolutePath()); - assertPathHasBeenCleared(endDir.toAbsolutePath()); + assertPathHasBeenCleared(newIndexDataPath.toAbsolutePath()); } public void testMaybeFlush() throws Exception { @@ -682,7 +673,7 @@ public static final IndexShard newIndexShard( } private static ShardRouting getInitializingShardRouting(ShardRouting existingShardRouting) { - ShardRouting shardRouting = TestShardRouting.newShardRouting(existingShardRouting.shardId(), + ShardRouting shardRouting = newShardRouting(existingShardRouting.shardId(), existingShardRouting.currentNodeId(), null, existingShardRouting.primary(), ShardRoutingState.INITIALIZING, existingShardRouting.allocationId()); shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"), @@ -919,4 +910,27 @@ public void testShardChangesWithDefaultDocType() throws Exception { assertThat(opsFromLucene, equalTo(opsFromTranslog)); } } + + /** + * Test that the {@link org.elasticsearch.index.engine.NoOpEngine} takes precedence over other + * engine factories if the index is closed. + */ + public void testNoOpEngineFactoryTakesPrecedence() { + final String indexName = "closed-index"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()); + ensureGreen(); + + assertAcked(client().admin().indices().prepareClose(indexName)); + + final ClusterService clusterService = getInstanceFromNode(ClusterService.class); + final ClusterState clusterState = clusterService.state(); + + final IndexMetaData indexMetaData = clusterState.metaData().index(indexName); + final IndicesService indicesService = getInstanceFromNode(IndicesService.class); + final IndexService indexService = indicesService.indexServiceSafe(indexMetaData.getIndex()); + + for (IndexShard indexShard : indexService) { + assertThat(indexShard.getEngine(), instanceOf(NoOpEngine.class)); + } + } } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java index 81cea988cd020..ac83c50fea6ae 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.indices; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -211,8 +212,13 @@ public void testIndexStateShardChanged() throws Throwable { assertThat(stateChangeListenerNode1.afterCloseSettings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1), equalTo(6)); assertThat(stateChangeListenerNode1.afterCloseSettings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1), equalTo(1)); - assertShardStatesMatch(stateChangeListenerNode1, 6, CLOSED); - assertShardStatesMatch(stateChangeListenerNode2, 6, CLOSED); + if (Version.CURRENT.onOrAfter(Version.V_8_0_0)) { + assertShardStatesMatch(stateChangeListenerNode1, 6, CLOSED, CREATED, RECOVERING, POST_RECOVERY, STARTED); + assertShardStatesMatch(stateChangeListenerNode2, 6, CLOSED, CREATED, RECOVERING, POST_RECOVERY, STARTED); + } else { + assertShardStatesMatch(stateChangeListenerNode1, 6, CLOSED); + assertShardStatesMatch(stateChangeListenerNode2, 6, CLOSED); + } } private static void assertShardStatesMatch(final IndexShardStateChangeListener stateChangeListener, diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index a8ae4bd11f61b..769d7a6c6866a 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -80,7 +80,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -272,57 +271,66 @@ public void testDeleteIndexStore() throws Exception { } public void testPendingTasks() throws Exception { - IndicesService indicesService = getIndicesService(); - IndexService test = createIndex("test"); + final IndexService indexService = createIndex("test"); + final Index index = indexService.index(); + final IndexSettings indexSettings = indexService.getIndexSettings(); - assertTrue(test.hasShard(0)); - ShardPath path = test.getShardOrNull(0).shardPath(); - assertTrue(test.getShardOrNull(0).routingEntry().started()); - ShardPath shardPath = ShardPath.loadShardPath(logger, getNodeEnvironment(), new ShardId(test.index(), 0), test.getIndexSettings()); - assertEquals(shardPath, path); - try { - indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS)); - fail("can't get lock"); - } catch (ShardLockObtainFailedException ex) { + final IndexShard indexShard = indexService.getShardOrNull(0); + assertNotNull(indexShard); + assertTrue(indexShard.routingEntry().started()); - } - assertTrue(path.exists()); + final ShardPath shardPath = indexShard.shardPath(); + assertEquals(ShardPath.loadShardPath(logger, getNodeEnvironment(), indexShard.shardId(), indexSettings), shardPath); + + final IndicesService indicesService = getIndicesService(); + expectThrows(ShardLockObtainFailedException.class, () -> + indicesService.processPendingDeletes(index, indexSettings, TimeValue.timeValueMillis(0))); + assertTrue(shardPath.exists()); int numPending = 1; if (randomBoolean()) { - indicesService.addPendingDelete(new ShardId(test.index(), 0), test.getIndexSettings()); + indicesService.addPendingDelete(indexShard.shardId(), indexSettings); } else { if (randomBoolean()) { numPending++; - indicesService.addPendingDelete(new ShardId(test.index(), 0), test.getIndexSettings()); + indicesService.addPendingDelete(indexShard.shardId(), indexSettings); } - indicesService.addPendingDelete(test.index(), test.getIndexSettings()); + indicesService.addPendingDelete(index, indexSettings); } + assertAcked(client().admin().indices().prepareClose("test")); - assertTrue(path.exists()); + assertTrue(shardPath.exists()); + ensureGreen("test"); - assertEquals(indicesService.numPendingDeletes(test.index()), numPending); + assertEquals(indicesService.numPendingDeletes(index), numPending); assertTrue(indicesService.hasUncompletedPendingDeletes()); - // shard lock released... we can now delete - indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS)); - assertEquals(indicesService.numPendingDeletes(test.index()), 0); - assertFalse(indicesService.hasUncompletedPendingDeletes()); - assertFalse(path.exists()); + expectThrows(ShardLockObtainFailedException.class, () -> + indicesService.processPendingDeletes(index, indexSettings, TimeValue.timeValueMillis(0))); - if (randomBoolean()) { - indicesService.addPendingDelete(new ShardId(test.index(), 0), test.getIndexSettings()); - indicesService.addPendingDelete(new ShardId(test.index(), 1), test.getIndexSettings()); - indicesService.addPendingDelete(new ShardId("bogus", "_na_", 1), test.getIndexSettings()); - assertEquals(indicesService.numPendingDeletes(test.index()), 2); + assertEquals(indicesService.numPendingDeletes(index), numPending); + assertTrue(indicesService.hasUncompletedPendingDeletes()); + + final boolean hasBogus = randomBoolean(); + if (hasBogus) { + indicesService.addPendingDelete(new ShardId(index, 0), indexSettings); + indicesService.addPendingDelete(new ShardId(index, 1), indexSettings); + indicesService.addPendingDelete(new ShardId("bogus", "_na_", 1), indexSettings); + assertEquals(indicesService.numPendingDeletes(index), numPending + 2); assertTrue(indicesService.hasUncompletedPendingDeletes()); - // shard lock released... we can now delete - indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS)); - assertEquals(indicesService.numPendingDeletes(test.index()), 0); - assertTrue(indicesService.hasUncompletedPendingDeletes()); // "bogus" index has not been removed } - assertAcked(client().admin().indices().prepareOpen("test").setTimeout(TimeValue.timeValueSeconds(1))); + assertAcked(client().admin().indices().prepareDelete("test")); + assertBusy(() -> { + try { + indicesService.processPendingDeletes(index, indexSettings, TimeValue.timeValueMillis(0)); + assertEquals(indicesService.numPendingDeletes(index), 0); + } catch (final Exception e) { + fail(e.getMessage()); + } + }); + assertThat(indicesService.hasUncompletedPendingDeletes(), equalTo(hasBogus)); // "bogus" index has not been removed + assertFalse(shardPath.exists()); } public void testVerifyIfIndexContentDeleted() throws Exception { @@ -551,7 +559,7 @@ public void testGetEngineFactory() throws IOException { } } - public void testConflictingEngineFactories() throws IOException { + public void testConflictingEngineFactories() { final String indexName = "foobar"; final Index index = new Index(indexName, UUIDs.randomBase64UUID()); final Settings settings = Settings.builder() diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index ea3e933a88314..82d6c38becaec 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -54,6 +54,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -209,24 +210,34 @@ public void testGatewayRecoveryTestActiveOnly() throws Exception { } public void testReplicaRecovery() throws Exception { - logger.info("--> start node A"); - String nodeA = internalCluster().startNode(); + final String nodeA = internalCluster().startNode(); + createIndex(INDEX_NAME, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, SHARD_COUNT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, REPLICA_COUNT) + .build()); + ensureGreen(INDEX_NAME); + + final int numOfDocs = scaledRandomIntBetween(0, 200); + try (BackgroundIndexer indexer = new BackgroundIndexer(INDEX_NAME, "_doc", client(), numOfDocs)) { + waitForDocs(numOfDocs, indexer); + } - logger.info("--> create index on node: {}", nodeA); - createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT); + refresh(INDEX_NAME); + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs); - logger.info("--> start node B"); - String nodeB = internalCluster().startNode(); - ensureGreen(); + final boolean closedIndex = randomBoolean(); + if (closedIndex) { + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + ensureGreen(INDEX_NAME); + } // force a shard recovery from nodeA to nodeB - logger.info("--> bump replica count"); - client().admin().indices().prepareUpdateSettings(INDEX_NAME) - .setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet(); - ensureGreen(); + final String nodeB = internalCluster().startNode(); + assertAcked(client().admin().indices().prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))); + ensureGreen(INDEX_NAME); - logger.info("--> request recoveries"); - RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet(); + final RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet(); // we should now have two total shards, one primary and one replica List recoveryStates = response.shardRecoveryStates().get(INDEX_NAME); @@ -238,14 +249,27 @@ public void testReplicaRecovery() throws Exception { assertThat(nodeBResponses.size(), equalTo(1)); // validate node A recovery - RecoveryState nodeARecoveryState = nodeAResponses.get(0); - assertRecoveryState(nodeARecoveryState, 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA); + final RecoveryState nodeARecoveryState = nodeAResponses.get(0); + final RecoverySource expectedRecoverySource; + if (closedIndex == false) { + expectedRecoverySource = RecoverySource.EmptyStoreRecoverySource.INSTANCE; + } else { + expectedRecoverySource = RecoverySource.ExistingStoreRecoverySource.INSTANCE; + } + assertRecoveryState(nodeARecoveryState, 0, expectedRecoverySource, true, Stage.DONE, null, nodeA); validateIndexRecoveryState(nodeARecoveryState.getIndex()); // validate node B recovery - RecoveryState nodeBRecoveryState = nodeBResponses.get(0); + final RecoveryState nodeBRecoveryState = nodeBResponses.get(0); assertRecoveryState(nodeBRecoveryState, 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeA, nodeB); validateIndexRecoveryState(nodeBRecoveryState.getIndex()); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA)); + + if (closedIndex) { + assertAcked(client().admin().indices().prepareOpen(INDEX_NAME)); + } + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs); } @TestLogging( diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java new file mode 100644 index 0000000000000..126c4df7928cd --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.indices.recovery; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.test.BackgroundIndexer; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; + +import java.util.Locale; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.is; + +@ESIntegTestCase.ClusterScope(numDataNodes = 2) +public class ReplicaToPrimaryPromotionIT extends ESIntegTestCase { + + @Override + protected int numberOfReplicas() { + return 1; + } + + public void testPromoteReplicaToPrimary() throws Exception { + final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + createIndex(indexName); + + final int numOfDocs = scaledRandomIntBetween(0, 200); + try (BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) { + waitForDocs(numOfDocs, indexer); + } + refresh(indexName); + + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs); + ensureGreen(indexName); + + // sometimes test with a closed index + final IndexMetaData.State indexState = randomFrom(IndexMetaData.State.OPEN, IndexMetaData.State.CLOSE); + if (indexState == IndexMetaData.State.CLOSE) { + assertAcked(client().admin().indices().prepareClose(indexName)); + ensureGreen(indexName); + } + + // pick up a data node that contains a random primary shard + ClusterState state = client(internalCluster().getMasterName()).admin().cluster().prepareState().get().getState(); + final int numShards = state.metaData().index(indexName).getNumberOfShards(); + final ShardRouting primaryShard = state.routingTable().index(indexName).shard(randomIntBetween(0, numShards - 1)).primaryShard(); + final DiscoveryNode randomNode = state.nodes().resolveNode(primaryShard.currentNodeId()); + + // stop the random data node, all remaining shards are promoted to primaries + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(randomNode.getName())); + ensureYellowAndNoInitializingShards(indexName); + + state = client(internalCluster().getMasterName()).admin().cluster().prepareState().get().getState(); + for (IndexShardRoutingTable shardRoutingTable : state.routingTable().index(indexName)) { + for (ShardRouting shardRouting : shardRoutingTable.activeShards()) { + assertThat(shardRouting + " should be promoted as a primary", shardRouting.primary(), is(true)); + } + } + + if (indexState == IndexMetaData.State.CLOSE) { + assertAcked(client().admin().indices().prepareOpen(indexName)); + ensureYellowAndNoInitializingShards(indexName); + } + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs); + } +} diff --git a/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java b/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java index 1d32283c6cb94..62d72c3f71954 100644 --- a/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java +++ b/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java @@ -20,9 +20,11 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.routing.ShardRouting; @@ -49,7 +51,6 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; public class CloseIndexIT extends ESIntegTestCase { @@ -113,7 +114,8 @@ public void testCloseAlreadyClosedIndex() throws Exception { assertIndexIsClosed(indexName); // Second close should be acked too - assertBusy(() -> assertAcked(client().admin().indices().prepareClose(indexName))); + final ActiveShardCount activeShardCount = randomFrom(ActiveShardCount.NONE, ActiveShardCount.DEFAULT, ActiveShardCount.ALL); + assertBusy(() -> assertAcked(client().admin().indices().prepareClose(indexName).setWaitForActiveShards(activeShardCount))); assertIndexIsClosed(indexName); } @@ -127,7 +129,7 @@ public void testCloseUnassignedIndex() throws Exception { assertThat(clusterState.metaData().indices().get(indexName).getState(), is(IndexMetaData.State.OPEN)); assertThat(clusterState.routingTable().allShards().stream().allMatch(ShardRouting::unassigned), is(true)); - assertBusy(() -> assertAcked(client().admin().indices().prepareClose(indexName))); + assertBusy(() -> assertAcked(client().admin().indices().prepareClose(indexName).setWaitForActiveShards(ActiveShardCount.NONE))); assertIndexIsClosed(indexName); } @@ -306,11 +308,34 @@ public void testConcurrentClosesAndOpens() throws Exception { indexer.totalIndexedDocs()); } + public void testCloseIndexWaitForActiveShards() throws Exception { + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + createIndex(indexName, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) // no replicas to avoid recoveries that could fail the index closing + .build()); + + final int nbDocs = randomIntBetween(0, 50); + indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, nbDocs) + .mapToObj(i -> client().prepareIndex(indexName, "_doc", String.valueOf(i)).setSource("num", i)).collect(toList())); + ensureGreen(indexName); + + final CloseIndexResponse closeIndexResponse = client().admin().indices().prepareClose(indexName).get(); + assertThat(client().admin().cluster().prepareHealth(indexName).get().getStatus(), is(ClusterHealthStatus.GREEN)); + assertTrue(closeIndexResponse.isAcknowledged()); + assertTrue(closeIndexResponse.isShardsAcknowledged()); + assertIndexIsClosed(indexName); + } + static void assertIndexIsClosed(final String... indices) { final ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); for (String index : indices) { - assertThat(clusterState.metaData().indices().get(index).getState(), is(IndexMetaData.State.CLOSE)); - assertThat(clusterState.routingTable().index(index), nullValue()); + final IndexMetaData indexMetaData = clusterState.metaData().indices().get(index); + assertThat(indexMetaData.getState(), is(IndexMetaData.State.CLOSE)); + final Settings indexSettings = indexMetaData.getSettings(); + assertThat(indexSettings.hasValue(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey()), is(true)); + assertThat(indexSettings.getAsBoolean(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey(), false), is(true)); + assertThat(clusterState.routingTable().index(index), notNullValue()); assertThat(clusterState.blocks().hasIndexBlock(index, MetaDataIndexStateService.INDEX_CLOSED_BLOCK), is(true)); assertThat("Index " + index + " must have only 1 block with [id=" + MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID + "]", clusterState.blocks().indices().getOrDefault(index, emptySet()).stream() @@ -321,7 +346,9 @@ static void assertIndexIsClosed(final String... indices) { static void assertIndexIsOpened(final String... indices) { final ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); for (String index : indices) { - assertThat(clusterState.metaData().indices().get(index).getState(), is(IndexMetaData.State.OPEN)); + final IndexMetaData indexMetaData = clusterState.metaData().indices().get(index); + assertThat(indexMetaData.getState(), is(IndexMetaData.State.OPEN)); + assertThat(indexMetaData.getSettings().hasValue(MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey()), is(false)); assertThat(clusterState.routingTable().index(index), notNullValue()); assertThat(clusterState.blocks().hasIndexBlock(index, MetaDataIndexStateService.INDEX_CLOSED_BLOCK), is(false)); } diff --git a/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java b/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java index 083c5ab1f5510..25d8f07bbd1cd 100644 --- a/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java +++ b/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java @@ -20,8 +20,8 @@ package org.elasticsearch.indices.state; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.close.TransportVerifyShardBeforeCloseAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -72,7 +72,7 @@ public void testReopenDuringClose() throws Exception { final CountDownLatch block = new CountDownLatch(1); final Releasable releaseBlock = interceptVerifyShardBeforeCloseActions(indexName, block::countDown); - ActionFuture closeIndexResponse = client().admin().indices().prepareClose(indexName).execute(); + ActionFuture closeIndexResponse = client().admin().indices().prepareClose(indexName).execute(); assertTrue("Waiting for index to have a closing blocked", block.await(60, TimeUnit.SECONDS)); assertIndexIsBlocked(indexName); assertFalse(closeIndexResponse.isDone()); @@ -96,7 +96,7 @@ public void testReopenDuringCloseOnMultipleIndices() throws Exception { final CountDownLatch block = new CountDownLatch(1); final Releasable releaseBlock = interceptVerifyShardBeforeCloseActions(randomFrom(indices), block::countDown); - ActionFuture closeIndexResponse = client().admin().indices().prepareClose("index-*").execute(); + ActionFuture closeIndexResponse = client().admin().indices().prepareClose("index-*").execute(); assertTrue("Waiting for index to have a closing blocked", block.await(60, TimeUnit.SECONDS)); assertFalse(closeIndexResponse.isDone()); indices.forEach(ReopenWhileClosingIT::assertIndexIsBlocked); diff --git a/server/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java b/server/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java index 1cc2d3e68e2ae..854dba7fb894b 100644 --- a/server/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java @@ -36,7 +36,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SimpleIndexStateIT extends ESIntegTestCase { @@ -65,7 +65,7 @@ public void testSimpleOpenClose() { stateResponse = client().admin().cluster().prepareState().get(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.CLOSE)); - assertThat(stateResponse.getState().routingTable().index("test"), nullValue()); + assertThat(stateResponse.getState().routingTable().index("test"), notNullValue()); logger.info("--> trying to index into a closed index ..."); try { @@ -102,7 +102,7 @@ public void testFastCloseAfterCreateContinuesCreateAfterOpen() { assertThat(health.isTimedOut(), equalTo(false)); assertThat(health.getStatus(), equalTo(ClusterHealthStatus.RED)); - assertAcked(client().admin().indices().prepareClose("test")); + assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(ActiveShardCount.NONE)); logger.info("--> updating test index settings to allow allocation"); client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() diff --git a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java index 9fb05af2040b3..e0ae78dff3466 100644 --- a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java +++ b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java @@ -521,11 +521,10 @@ public void testStringSortMissingAscTerminates() throws Exception { assertThat(response.getHits().getHits().length, equalTo(0)); } - public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException { + public void testCloseAndReopenOrDeleteWithActiveScroll() { createIndex("test"); for (int i = 0; i < 100; i++) { - client().prepareIndex("test", "type1", Integer.toString(i)).setSource(jsonBuilder().startObject().field("field", i).endObject()) - .get(); + client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field", i).get(); } refresh(); SearchResponse searchResponse = client().prepareSearch() @@ -541,11 +540,11 @@ public void testCloseAndReopenOrDeleteWithActiveScroll() throws IOException { assertThat(((Number) hit.getSortValues()[0]).longValue(), equalTo(counter++)); } if (randomBoolean()) { - client().admin().indices().prepareClose("test").get(); - client().admin().indices().prepareOpen("test").get(); + assertAcked(client().admin().indices().prepareClose("test")); + assertAcked(client().admin().indices().prepareOpen("test")); ensureGreen("test"); } else { - client().admin().indices().prepareDelete("test").get(); + assertAcked(client().admin().indices().prepareDelete("test")); } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 9646ebf1f1aa5..ac21b651f51d2 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1556,7 +1556,7 @@ public void testSnapshotClosedIndex() throws Exception { assertAcked(client.admin().indices().prepareClose("test-idx-closed")); ClusterStateResponse stateResponse = client.admin().cluster().prepareState().get(); assertThat(stateResponse.getState().metaData().index("test-idx-closed").getState(), equalTo(IndexMetaData.State.CLOSE)); - assertThat(stateResponse.getState().routingTable().index("test-idx-closed"), nullValue()); + assertThat(stateResponse.getState().routingTable().index("test-idx-closed"), notNullValue()); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -3751,7 +3751,7 @@ public void testRestoreIncreasesPrimaryTerms() { final IndexMetaData restoredIndexMetaData = client().admin().cluster().prepareState().clear().setIndices(indexName) .setMetaData(true).get().getState().metaData().index(indexName); for (int shardId = 0; shardId < numPrimaries; shardId++) { - assertThat(restoredIndexMetaData.primaryTerm(shardId), equalTo(primaryTerms.get(shardId) + 1)); + assertThat(restoredIndexMetaData.primaryTerm(shardId), greaterThan(primaryTerms.get(shardId))); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index e7b3f39747124..f6731f1e2ad93 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -697,6 +697,14 @@ public EngineConfig config( tombstoneDocSupplier()); } + protected EngineConfig noOpConfig(IndexSettings indexSettings, Store store, Path translogPath) { + return noOpConfig(indexSettings, store, translogPath, null); + } + + protected EngineConfig noOpConfig(IndexSettings indexSettings, Store store, Path translogPath, LongSupplier globalCheckpointSupplier) { + return config(indexSettings, store, translogPath, newMergePolicy(), null, null, globalCheckpointSupplier); + } + protected static final BytesReference B_1 = new BytesArray(new byte[]{1}); protected static final BytesReference B_2 = new BytesArray(new byte[]{2}); protected static final BytesReference B_3 = new BytesArray(new byte[]{3}); diff --git a/x-pack/plugin/ccr/build.gradle b/x-pack/plugin/ccr/build.gradle index b8ed9f55932cc..e1ddb2f12d78b 100644 --- a/x-pack/plugin/ccr/build.gradle +++ b/x-pack/plugin/ccr/build.gradle @@ -18,6 +18,9 @@ integTest.enabled = false compileJava.options.compilerArgs << "-Xlint:-try" compileTestJava.options.compilerArgs << "-Xlint:-try" +// Integration Test classes that cannot run with the security manager +String[] noSecurityManagerITClasses = [ "**/CloseFollowerIndexIT.class" ] + // Instead we create a separate task to run the // tests based on ESIntegTestCase task internalClusterTest(type: RandomizedTestingTask, @@ -25,11 +28,22 @@ task internalClusterTest(type: RandomizedTestingTask, description: 'Java fantasy integration tests', dependsOn: unitTest.dependsOn) { include '**/*IT.class' + exclude noSecurityManagerITClasses systemProperty 'es.set.netty.runtime.available.processors', 'false' } check.dependsOn internalClusterTest internalClusterTest.mustRunAfter test +task internalClusterTestNoSecurityManager(type: RandomizedTestingTask, + group: JavaBasePlugin.VERIFICATION_GROUP, + description: 'Java fantasy integration tests with no security manager', + dependsOn: unitTest.dependsOn) { + include noSecurityManagerITClasses + systemProperty 'es.set.netty.runtime.available.processors', 'false' + systemProperty 'tests.security.manager', 'false' +} +internalClusterTest.dependsOn internalClusterTestNoSecurityManager + // add all sub-projects of the qa sub-project gradle.projectsEvaluated { project.subprojects diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index 81d8750d07c6d..595303d0bce72 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; @@ -204,7 +205,7 @@ private void closeIndexUpdateSettingsAndOpenIndex(String followIndex, Runnable handler, Consumer onFailure) { CloseIndexRequest closeRequest = new CloseIndexRequest(followIndex); - CheckedConsumer onResponse = response -> { + CheckedConsumer onResponse = response -> { updateSettingsAndOpenIndex(followIndex, updatedSettings, handler, onFailure); }; followerClient.admin().indices().close(closeRequest, ActionListener.wrap(onResponse, onFailure)); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java index 0551d30c2e73a..7f93934fd91f8 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java @@ -11,17 +11,22 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.ReadOnlyEngine; import org.elasticsearch.xpack.CcrIntegTestCase; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; +import org.junit.After; +import org.junit.Before; -import java.util.ArrayList; -import java.util.List; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static java.util.Collections.singletonMap; @@ -31,6 +36,36 @@ public class CloseFollowerIndexIT extends CcrIntegTestCase { + private Thread.UncaughtExceptionHandler uncaughtExceptionHandler; + + @Before + public void wrapUncaughtExceptionHandler() { + uncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); + AccessController.doPrivileged((PrivilegedAction) () -> { + Thread.setDefaultUncaughtExceptionHandler((t, e) -> { + if (t.getThreadGroup().getName().contains(getTestClass().getSimpleName())) { + for (StackTraceElement element : e.getStackTrace()) { + if (element.getClassName().equals(ReadOnlyEngine.class.getName())) { + if (element.getMethodName().equals("assertMaxSeqNoEqualsToGlobalCheckpoint")) { + return; + } + } + } + } + uncaughtExceptionHandler.uncaughtException(t, e); + }); + return null; + }); + } + + @After + public void restoreUncaughtExceptionHandler() { + AccessController.doPrivileged((PrivilegedAction) () -> { + Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler); + return null; + }); + } + public void testCloseAndReopenFollowerIndex() throws Exception { final String leaderIndexSettings = getIndexSettings(1, 1, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); @@ -62,20 +97,29 @@ public void testCloseAndReopenFollowerIndex() throws Exception { } atLeastDocsIndexed(followerClient(), "index2", 32); - AcknowledgedResponse response = followerClient().admin().indices().close(new CloseIndexRequest("index2")).get(); + + CloseIndexRequest closeIndexRequest = new CloseIndexRequest("index2"); + closeIndexRequest.waitForActiveShards(ActiveShardCount.NONE); + AcknowledgedResponse response = followerClient().admin().indices().close(closeIndexRequest).get(); assertThat(response.isAcknowledged(), is(true)); ClusterState clusterState = followerClient().admin().cluster().prepareState().get().getState(); - List blocks = new ArrayList<>(clusterState.getBlocks().indices().get("index2")); - assertThat(blocks.size(), equalTo(1)); - assertThat(blocks.get(0).id(), equalTo(MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID)); + assertThat(clusterState.metaData().index("index2").getState(), is(IndexMetaData.State.CLOSE)); + assertThat(clusterState.getBlocks().hasIndexBlock("index2", MetaDataIndexStateService.INDEX_CLOSED_BLOCK), is(true)); + assertThat(followerClient().admin().cluster().prepareHealth("index2").get().getStatus(), equalTo(ClusterHealthStatus.RED)); isRunning.set(false); for (Thread thread : threads) { thread.join(); } + assertAcked(followerClient().admin().indices().open(new OpenIndexRequest("index2")).get()); + clusterState = followerClient().admin().cluster().prepareState().get().getState(); + assertThat(clusterState.metaData().index("index2").getState(), is(IndexMetaData.State.OPEN)); + assertThat(clusterState.getBlocks().hasIndexBlockWithId("index2", MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID), is(false)); + ensureFollowerGreen("index2"); + refresh(leaderClient(), "index1"); SearchRequest leaderSearchRequest = new SearchRequest("index1"); leaderSearchRequest.source().trackTotalHits(true); @@ -86,6 +130,6 @@ public void testCloseAndReopenFollowerIndex() throws Exception { followerSearchRequest.source().trackTotalHits(true); long followerIndexDocs = followerClient().search(followerSearchRequest).actionGet().getHits().getTotalHits().value; assertThat(followerIndexDocs, equalTo(leaderIndexDocs)); - }); + }, 30L, TimeUnit.SECONDS); } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index d9b75f416b38d..52202982701c1 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -942,7 +942,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception { } assertBusy(() -> { - assertThat(getFollowTaskSettingsVersion("follower"), equalTo(2L)); + assertThat(getFollowTaskSettingsVersion("follower"), equalTo(4L)); assertThat(getFollowTaskMappingVersion("follower"), equalTo(2L)); GetSettingsRequest getSettingsRequest = new GetSettingsRequest(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java index 1efe5389d81b2..91b91ddd04f3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; import org.elasticsearch.action.admin.indices.open.OpenIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; @@ -126,9 +127,9 @@ protected void masterOperation(Task task, TransportFreezeIndexAction.FreezeReque .masterNodeTimeout(request.masterNodeTimeout()) .indices(concreteIndices); - indexStateService.closeIndices(closeRequest, new ActionListener() { + indexStateService.closeIndices(closeRequest, new ActionListener() { @Override - public void onResponse(final AcknowledgedResponse response) { + public void onResponse(final CloseIndexResponse response) { if (response.isAcknowledged()) { toggleFrozenSettings(concreteIndices, request, listener); } else { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java index 983b186c4ccf6..9231bad9a8dfe 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java @@ -349,7 +349,7 @@ public void testFreezeIndexIncreasesIndexSettingsVersion() throws ExecutionExcep assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest(index))); assertIndexFrozen(index); assertThat(client().admin().cluster().prepareState().get().getState().metaData().index(index).getSettingsVersion(), - equalTo(settingsVersion + 1)); + greaterThan(settingsVersion)); } public void testFreezeEmptyIndexWithTranslogOps() throws Exception { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java index 25e1c4e481bba..368afaa26d0cc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java @@ -8,7 +8,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; @@ -43,8 +43,8 @@ public void testCloseFollowingIndex() { CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onResponse(new AcknowledgedResponse(true)); + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new CloseIndexResponse(true, true)); return null; }).when(indicesClient).close(Mockito.any(), Mockito.any());