From efbd18d4ba7e24affb3213dc3ccfe27cb495cce6 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 15:16:54 -0400 Subject: [PATCH 01/12] Organize imports --- .../elasticsearch/cluster/metadata/Metadata.java | 14 +++++++------- .../MetadataMigrateToDataTiersRoutingService.java | 4 ++-- ...adataMigrateToDataTiersRoutingServiceTests.java | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 754ff3511eecc..6a8df01529b66 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -38,14 +38,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.NamedObjectNotFoundException; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.ToXContentFragment; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.gateway.MetadataStateFormat; @@ -53,6 +46,13 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.NamedObjectNotFoundException; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index 7f5edb8d1489b..83ef0e83c5a9c 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -15,13 +15,13 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ilm.AllocateAction; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.ilm.LifecycleAction; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java index f97bc1d376910..4a626c2698a11 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java @@ -17,13 +17,13 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.MigratedEntities; import org.elasticsearch.xpack.core.ilm.AllocateAction; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; From a05bde35a703923f758a5a572fa7343b13e7da69 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 15:17:16 -0400 Subject: [PATCH 02/12] Slightly better ergonomics No need to gen up a single element array if you have an index name --- .../main/java/org/elasticsearch/cluster/metadata/Metadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 6a8df01529b66..bb996083220ad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -426,7 +426,7 @@ public ImmutableOpenMap findMappings(String[] concreteI /** * Finds the parent data streams, if any, for the specified concrete indices. */ - public ImmutableOpenMap findDataStreams(String[] concreteIndices) { + public ImmutableOpenMap findDataStreams(String... concreteIndices) { assert concreteIndices != null; final ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); final SortedMap lookup = getIndicesLookup(); From 582714c51d06bc9104c72b75f39c5d1ea6a98706 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 15:19:07 -0400 Subject: [PATCH 03/12] Whitespace --- ...adataMigrateToDataTiersRoutingService.java | 2 +- ...MigrateToDataTiersRoutingServiceTests.java | 78 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index 83ef0e83c5a9c..abcf4fed7bbd3 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -428,7 +428,7 @@ private static Settings maybeMigrateRoutingSettingToTierPreference(String attrib newSettingsBuilder.remove(attributeBasedRoutingSettingName); logger.debug("index [{}]: removed setting [{}]", indexName, attributeBasedRoutingSettingName); logger.debug("index [{}]: configured setting [{}] to [{}]", indexName, - TIER_PREFERENCE, convertedTierPreference); + TIER_PREFERENCE, convertedTierPreference); } else { // log warning and do *not* remove setting, return the settings unchanged logger.warn("index [{}]: could not convert attribute based setting [{}] value of [{}] to a tier preference " + diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java index 4a626c2698a11..dad13b3ee8b0e 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java @@ -97,9 +97,9 @@ public void testMigrateIlmPolicyForIndexWithoutILMMetadata() { coldAllocateAction); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) - .put(IndexMetadata.builder(indexName).settings(getBaseIndexSettings())).build()) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) + .put(IndexMetadata.builder(indexName).settings(getBaseIndexSettings())).build()) .build(); Metadata.Builder newMetadata = Metadata.builder(state.metadata()); @@ -138,9 +138,9 @@ public void testMigrateIlmPolicyFOrPhaseWithDeactivatedMigrateAction() { randomNonNegativeLong(), randomNonNegativeLong()); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) - .put(IndexMetadata.builder(indexName).settings(getBaseIndexSettings())).build()) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) + .put(IndexMetadata.builder(indexName).settings(getBaseIndexSettings())).build()) .build(); Metadata.Builder newMetadata = Metadata.builder(state.metadata()); @@ -153,7 +153,7 @@ public void testMigrateIlmPolicyFOrPhaseWithDeactivatedMigrateAction() { LifecyclePolicy lifecyclePolicy = updatedLifecycleMetadata.getPolicies().get(lifecycleName); Map warmActions = lifecyclePolicy.getPhases().get("warm").getActions(); assertThat("allocate action in the warm phase didn't specify any number of replicas so it must be removed, together with the " + - "deactivated migrate action", warmActions.size(), is(1)); + "deactivated migrate action", warmActions.size(), is(1)); assertThat(warmActions.get(shrinkAction.getWriteableName()), is(shrinkAction)); } @@ -179,9 +179,9 @@ public void testMigrateIlmPolicyRefreshesCachedPhase() { .putCustom(ILM_CUSTOM_METADATA_KEY, preMigrationExecutionState.asMap()); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) - .put(indexMetadata).build()) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) + .put(indexMetadata).build()) .build(); Metadata.Builder newMetadata = Metadata.builder(state.metadata()); @@ -217,9 +217,9 @@ public void testMigrateIlmPolicyRefreshesCachedPhase() { .putCustom(ILM_CUSTOM_METADATA_KEY, preMigrationExecutionState.asMap()); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) - .put(indexMetadata).build()) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) + .put(indexMetadata).build()) .build(); Metadata.Builder newMetadata = Metadata.builder(state.metadata()); @@ -259,9 +259,9 @@ public void testMigrateIlmPolicyRefreshesCachedPhase() { .putCustom(ILM_CUSTOM_METADATA_KEY, preMigrationExecutionState.asMap()); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) - .put(indexMetadata).build()) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) + .put(indexMetadata).build()) .build(); Metadata.Builder newMetadata = Metadata.builder(state.metadata()); @@ -299,9 +299,9 @@ public void testMigrateIlmPolicyRefreshesCachedPhase() { .putCustom(ILM_CUSTOM_METADATA_KEY, preMigrationExecutionState.asMap()); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) - .put(indexMetadata).build()) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.STOPPED)) + .put(indexMetadata).build()) .build(); Metadata.Builder newMetadata = Metadata.builder(state.metadata()); @@ -342,7 +342,7 @@ public void testAllocateActionDefinesRoutingRules() { assertThat(allocateActionDefinesRoutingRules("data", new AllocateAction(null, null, Map.of("another_attribute", "rack1"), null, Map.of("data", "cold"))), is(true)); assertThat(allocateActionDefinesRoutingRules("data", new AllocateAction(null, null, null, null, Map.of("another_attribute", - "cold"))), + "cold"))), is(false)); assertThat(allocateActionDefinesRoutingRules("data", null), is(false)); } @@ -574,16 +574,16 @@ public void testMigrateToDataTiersRouting() { IndexMetadata.builder("indexWitWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, "warm")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Map.of(policyToMigrate.getName(), policyWithDataAttribute, shouldntBeMigratedPolicy.getName(), policyWithOtherAttribute), - OperationMode.STOPPED)) - .put(IndexTemplateMetadata.builder("catch-all").patterns(List.of("*")) - .settings(Settings.builder().put(DATA_ROUTING_REQUIRE_SETTING, "hot")) - .build()) - .put(IndexTemplateMetadata.builder("other-template").patterns(List.of("other-*")) - .settings(Settings.builder().put(DATA_ROUTING_REQUIRE_SETTING, "hot")) - .build()) - .put(indexWithUnknownDataAttribute).put(indexWitWarmDataAttribute)) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Map.of(policyToMigrate.getName(), policyWithDataAttribute, shouldntBeMigratedPolicy.getName(), policyWithOtherAttribute), + OperationMode.STOPPED)) + .put(IndexTemplateMetadata.builder("catch-all").patterns(List.of("*")) + .settings(Settings.builder().put(DATA_ROUTING_REQUIRE_SETTING, "hot")) + .build()) + .put(IndexTemplateMetadata.builder("other-template").patterns(List.of("other-*")) + .settings(Settings.builder().put(DATA_ROUTING_REQUIRE_SETTING, "hot")) + .build()) + .put(indexWithUnknownDataAttribute).put(indexWitWarmDataAttribute)) .build(); { @@ -640,8 +640,8 @@ public void testMigrateToDataTiersRouting() { public void testMigrateToDataTiersRoutingRequiresILMStopped() { { ClusterState ilmRunningState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Map.of(), OperationMode.RUNNING))) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Map.of(), OperationMode.RUNNING))) .build(); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, () -> migrateToDataTiersRouting(ilmRunningState, "data", "catch-all", REGISTRY, client, null)); @@ -650,8 +650,8 @@ public void testMigrateToDataTiersRoutingRequiresILMStopped() { { ClusterState ilmStoppingState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Map.of(), OperationMode.STOPPING))) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Map.of(), OperationMode.STOPPING))) .build(); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, () -> migrateToDataTiersRouting(ilmStoppingState, "data", "catch-all", REGISTRY, client, null)); @@ -660,8 +660,8 @@ public void testMigrateToDataTiersRoutingRequiresILMStopped() { { ClusterState ilmStoppedState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Map.of(), OperationMode.STOPPED))) + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Map.of(), OperationMode.STOPPED))) .build(); Tuple migratedState = migrateToDataTiersRouting(ilmStoppedState, "data", "catch-all", REGISTRY, client, null); @@ -679,7 +679,7 @@ public void testMigrationDoesNotRemoveComposableTemplates() { String composableTemplateName = "catch-all-composable-template"; ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() - .put(composableTemplateName, composableIndexTemplate).build()) + .put(composableTemplateName, composableIndexTemplate).build()) .build(); Tuple migratedEntitiesTuple = migrateToDataTiersRouting(clusterState, "data", composableTemplateName, REGISTRY, client, null); @@ -748,8 +748,8 @@ private String getColdPhaseDefinition() { private Map getPhaseDefinitionAsMap(LifecycleExecutionState newLifecycleState) { XContentType entityContentType = XContentType.fromMediaType("application/json"); return (Map) XContentHelper.convertToMap(entityContentType.xContent(), - new ByteArrayInputStream(newLifecycleState.getPhaseDefinition().getBytes(StandardCharsets.UTF_8)), - false) + new ByteArrayInputStream(newLifecycleState.getPhaseDefinition().getBytes(StandardCharsets.UTF_8)), + false) .get("phase_definition"); } From c8964c95c831505edb5e212eacf5c581924aed29 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 15:20:39 -0400 Subject: [PATCH 04/12] Fix a typo -- indexWit*h*WarmDataAttribute --- ...MigrateToDataTiersRoutingServiceTests.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java index dad13b3ee8b0e..ceda491239494 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java @@ -359,40 +359,40 @@ public void testConvertAttributeValueToTierPreference() { public void testMigrateIndices() { { // index with `warm` data attribute is migrated to the equivalent _tier_preference routing - IndexMetadata.Builder indexWitWarmDataAttribute = - IndexMetadata.builder("indexWitWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, + IndexMetadata.Builder indexWithWarmDataAttribute = + IndexMetadata.builder("indexWithWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, "warm")); ClusterState state = - ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWitWarmDataAttribute)).build(); + ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWithWarmDataAttribute)).build(); Metadata.Builder mb = Metadata.builder(state.metadata()); List migratedIndices = migrateIndices(mb, state, "data"); assertThat(migratedIndices.size(), is(1)); - assertThat(migratedIndices.get(0), is("indexWitWarmDataAttribute")); + assertThat(migratedIndices.get(0), is("indexWithWarmDataAttribute")); ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build(); - IndexMetadata migratedIndex = migratedState.metadata().index("indexWitWarmDataAttribute"); + IndexMetadata migratedIndex = migratedState.metadata().index("indexWithWarmDataAttribute"); assertThat(migratedIndex.getSettings().get(DATA_ROUTING_REQUIRE_SETTING), nullValue()); assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_warm,data_hot")); } { // test the migration of the `include.data` configuration to the equivalent _tier_preference routing - IndexMetadata.Builder indexWitWarmDataAttribute = - IndexMetadata.builder("indexWitWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_INCLUDE_SETTING, + IndexMetadata.Builder indexWithWarmDataAttribute = + IndexMetadata.builder("indexWithWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_INCLUDE_SETTING, "warm")); ClusterState state = - ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWitWarmDataAttribute)).build(); + ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWithWarmDataAttribute)).build(); Metadata.Builder mb = Metadata.builder(state.metadata()); List migratedIndices = migrateIndices(mb, state, "data"); assertThat(migratedIndices.size(), is(1)); - assertThat(migratedIndices.get(0), is("indexWitWarmDataAttribute")); + assertThat(migratedIndices.get(0), is("indexWithWarmDataAttribute")); ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build(); - IndexMetadata migratedIndex = migratedState.metadata().index("indexWitWarmDataAttribute"); + IndexMetadata migratedIndex = migratedState.metadata().index("indexWithWarmDataAttribute"); assertThat(migratedIndex.getSettings().get(DATA_ROUTING_INCLUDE_SETTING), nullValue()); assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_warm,data_hot")); } @@ -570,8 +570,8 @@ public void testMigrateToDataTiersRouting() { IndexMetadata.Builder indexWithUnknownDataAttribute = IndexMetadata.builder("indexWithUnknownDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, "something_else")); - IndexMetadata.Builder indexWitWarmDataAttribute = - IndexMetadata.builder("indexWitWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, "warm")); + IndexMetadata.Builder indexWithWarmDataAttribute = + IndexMetadata.builder("indexWithWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, "warm")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( @@ -583,7 +583,7 @@ public void testMigrateToDataTiersRouting() { .put(IndexTemplateMetadata.builder("other-template").patterns(List.of("other-*")) .settings(Settings.builder().put(DATA_ROUTING_REQUIRE_SETTING, "hot")) .build()) - .put(indexWithUnknownDataAttribute).put(indexWitWarmDataAttribute)) + .put(indexWithUnknownDataAttribute).put(indexWithWarmDataAttribute)) .build(); { @@ -595,7 +595,7 @@ public void testMigrateToDataTiersRouting() { assertThat(migratedEntities.migratedPolicies.size(), is(1)); assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); assertThat(migratedEntities.migratedIndices.size(), is(1)); - assertThat(migratedEntities.migratedIndices.get(0), is("indexWitWarmDataAttribute")); + assertThat(migratedEntities.migratedIndices.get(0), is("indexWithWarmDataAttribute")); ClusterState newState = migratedEntitiesTuple.v1(); assertThat(newState.metadata().getTemplates().size(), is(1)); @@ -613,7 +613,7 @@ public void testMigrateToDataTiersRouting() { assertThat(migratedEntities.migratedPolicies.size(), is(1)); assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); assertThat(migratedEntities.migratedIndices.size(), is(1)); - assertThat(migratedEntities.migratedIndices.get(0), is("indexWitWarmDataAttribute")); + assertThat(migratedEntities.migratedIndices.get(0), is("indexWithWarmDataAttribute")); ClusterState newState = migratedEntitiesTuple.v1(); assertThat(newState.metadata().getTemplates().size(), is(2)); @@ -630,9 +630,9 @@ public void testMigrateToDataTiersRouting() { assertThat(migratedEntities.migratedPolicies.size(), is(1)); assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); assertThat(migratedEntities.migratedIndices.size(), is(1)); - assertThat(migratedEntities.migratedIndices.get(0), is("indexWitWarmDataAttribute")); + assertThat(migratedEntities.migratedIndices.get(0), is("indexWithWarmDataAttribute")); - IndexMetadata migratedIndex = migratedEntitiesTuple.v1().metadata().index("indexWitWarmDataAttribute"); + IndexMetadata migratedIndex = migratedEntitiesTuple.v1().metadata().index("indexWithWarmDataAttribute"); assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_warm,data_hot")); } } From 19c60d45265271cafa296449d135a49fd00438cc Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 15:28:55 -0400 Subject: [PATCH 05/12] More whitespace --- ...MigrateToDataTiersRoutingServiceTests.java | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java index ceda491239494..1c757af351ffd 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java @@ -360,8 +360,8 @@ public void testMigrateIndices() { { // index with `warm` data attribute is migrated to the equivalent _tier_preference routing IndexMetadata.Builder indexWithWarmDataAttribute = - IndexMetadata.builder("indexWithWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, - "warm")); + IndexMetadata.builder("indexWithWarmDataAttribute").settings(getBaseIndexSettings() + .put(DATA_ROUTING_REQUIRE_SETTING, "warm")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWithWarmDataAttribute)).build(); @@ -380,8 +380,8 @@ public void testMigrateIndices() { { // test the migration of the `include.data` configuration to the equivalent _tier_preference routing IndexMetadata.Builder indexWithWarmDataAttribute = - IndexMetadata.builder("indexWithWarmDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_INCLUDE_SETTING, - "warm")); + IndexMetadata.builder("indexWithWarmDataAttribute").settings(getBaseIndexSettings() + .put(DATA_ROUTING_INCLUDE_SETTING, "warm")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWithWarmDataAttribute)).build(); @@ -404,8 +404,7 @@ public void testMigrateIndices() { IndexMetadata.builder("indexWithTierPreferenceAndDataAttribute").settings(getBaseIndexSettings() .put(DATA_ROUTING_REQUIRE_SETTING, "cold") .put(DATA_ROUTING_INCLUDE_SETTING, "hot") - .put(TIER_PREFERENCE, "data_warm,data_hot") - ); + .put(TIER_PREFERENCE, "data_warm,data_hot")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWithTierPreferenceAndDataAttribute)).build(); @@ -428,8 +427,7 @@ public void testMigrateIndices() { IndexMetadata.Builder indexWithTierPreferenceAndDataAttribute = IndexMetadata.builder("indexWithTierPreferenceAndDataAttribute").settings(getBaseIndexSettings() .put(DATA_ROUTING_INCLUDE_SETTING, "cold") - .put(TIER_PREFERENCE, "data_warm,data_hot") - ); + .put(TIER_PREFERENCE, "data_warm,data_hot")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWithTierPreferenceAndDataAttribute)).build(); @@ -448,8 +446,8 @@ public void testMigrateIndices() { { // index with an unknown `data` attribute routing value should **not** be migrated IndexMetadata.Builder indexWithUnknownDataAttribute = - IndexMetadata.builder("indexWithUnknownDataAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, - "something_else")); + IndexMetadata.builder("indexWithUnknownDataAttribute").settings(getBaseIndexSettings() + .put(DATA_ROUTING_REQUIRE_SETTING, "something_else")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexWithUnknownDataAttribute)).build(); @@ -466,9 +464,9 @@ public void testMigrateIndices() { // index with data and another attribute should only see the data attribute removed and the corresponding tier_preference // configured IndexMetadata.Builder indexDataAndBoxAttribute = - IndexMetadata.builder("indexWithDataAndBoxAttribute").settings(getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, - "warm").put(BOX_ROUTING_REQUIRE_SETTING, "box1")); - + IndexMetadata.builder("indexWithDataAndBoxAttribute").settings(getBaseIndexSettings() + .put(DATA_ROUTING_REQUIRE_SETTING, "warm") + .put(BOX_ROUTING_REQUIRE_SETTING, "box1")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexDataAndBoxAttribute)).build(); @@ -487,8 +485,8 @@ public void testMigrateIndices() { { // index that doesn't have any data attribute routing but has another attribute should not see any change IndexMetadata.Builder indexBoxAttribute = - IndexMetadata.builder("indexWithBoxAttribute").settings(getBaseIndexSettings().put(BOX_ROUTING_REQUIRE_SETTING, "warm")); - + IndexMetadata.builder("indexWithBoxAttribute").settings(getBaseIndexSettings() + .put(BOX_ROUTING_REQUIRE_SETTING, "warm")); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexBoxAttribute)).build(); @@ -506,7 +504,6 @@ public void testMigrateIndices() { { IndexMetadata.Builder indexNoRoutingAttribute = IndexMetadata.builder("indexNoRoutingAttribute").settings(getBaseIndexSettings()); - ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexNoRoutingAttribute)).build(); @@ -575,7 +572,8 @@ public void testMigrateToDataTiersRouting() { ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder() .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( - Map.of(policyToMigrate.getName(), policyWithDataAttribute, shouldntBeMigratedPolicy.getName(), policyWithOtherAttribute), + Map.of(policyToMigrate.getName(), policyWithDataAttribute, + shouldntBeMigratedPolicy.getName(), policyWithOtherAttribute), OperationMode.STOPPED)) .put(IndexTemplateMetadata.builder("catch-all").patterns(List.of("*")) .settings(Settings.builder().put(DATA_ROUTING_REQUIRE_SETTING, "hot")) From 6488fac3ea064660a9a9e609d65fa517ab76ba7f Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 15:30:37 -0400 Subject: [PATCH 06/12] Move and reword comments, cleanup --- .../MetadataMigrateToDataTiersRoutingService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index abcf4fed7bbd3..3e0d6a718cc10 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -376,9 +376,11 @@ static List migrateIndices(Metadata.Builder mb, ClusterState currentStat for (ObjectObjectCursor index : currentState.metadata().indices()) { IndexMetadata indexMetadata = index.value; Settings currentSettings = indexMetadata.getSettings(); + // migrate using the `require` setting Settings newSettings = maybeMigrateRoutingSettingToTierPreference(nodeAttrIndexRequireRoutingSetting, indexMetadata); + if (newSettings.equals(currentSettings)) { - // migrating based on the `require` setting was not successful so let's check if the index used the `include` routing + // migrating based on the `require` setting was not successful, so let's check if the index used the `include` routing // setting to configure the allocations and try to migrate it newSettings = maybeMigrateRoutingSettingToTierPreference(nodeAttrIndexIncludeRoutingSetting, indexMetadata); } @@ -413,9 +415,11 @@ private static Settings maybeMigrateRoutingSettingToTierPreference(String attrib if (currentIndexSettings.keySet().contains(attributeBasedRoutingSettingName) == false) { return currentIndexSettings; } - // look at the value, get the correct tiers config and update the settings and index metadata + Settings.Builder newSettingsBuilder = Settings.builder().put(currentIndexSettings); String indexName = indexMetadata.getIndex().getName(); + + // look at the value, get the correct tiers config and update the settings if (currentIndexSettings.keySet().contains(TIER_PREFERENCE)) { newSettingsBuilder.remove(attributeBasedRoutingSettingName); logger.debug("index [{}]: removed setting [{}]", indexName, attributeBasedRoutingSettingName); From d1002dd84934e3c4d881e1bdf04a121eccf09a9e Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 15:31:26 -0400 Subject: [PATCH 07/12] Add a backstop that always sets the TIER_PREFERENCE --- ...adataMigrateToDataTiersRoutingService.java | 39 ++++++++++++++++--- ...MigrateToDataTiersRoutingServiceTests.java | 33 +++++++++------- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index 3e0d6a718cc10..687d4bb723bc8 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -376,6 +376,9 @@ static List migrateIndices(Metadata.Builder mb, ClusterState currentStat for (ObjectObjectCursor index : currentState.metadata().indices()) { IndexMetadata indexMetadata = index.value; Settings currentSettings = indexMetadata.getSettings(); + + boolean removeNodeAttrIndexRoutingSettings = true; + // migrate using the `require` setting Settings newSettings = maybeMigrateRoutingSettingToTierPreference(nodeAttrIndexRequireRoutingSetting, indexMetadata); @@ -384,14 +387,23 @@ static List migrateIndices(Metadata.Builder mb, ClusterState currentStat // setting to configure the allocations and try to migrate it newSettings = maybeMigrateRoutingSettingToTierPreference(nodeAttrIndexIncludeRoutingSetting, indexMetadata); } + if (newSettings.equals(currentSettings)) { + removeNodeAttrIndexRoutingSettings = false; + // migrating based on the `include` setting was not successful, + // so, last stop, we just inject a tier preference regardless of anything else + newSettings = migrateToDefaultTierPreference(currentState, indexMetadata); + } if (newSettings.equals(currentSettings) == false) { - // we converted either the require or the include routing setting to tier preference - // so let's clear all the routing settings for the given attribute Settings.Builder finalSettings = Settings.builder().put(newSettings); - finalSettings.remove(nodeAttrIndexExcludeRoutingSetting); - finalSettings.remove(nodeAttrIndexRequireRoutingSetting); - finalSettings.remove(nodeAttrIndexIncludeRoutingSetting); + + if (removeNodeAttrIndexRoutingSettings) { + // we converted either the `require` or the `include` routing setting to tier preference + // so let's clear all the routing settings for the given attribute + finalSettings.remove(nodeAttrIndexExcludeRoutingSetting); + finalSettings.remove(nodeAttrIndexRequireRoutingSetting); + finalSettings.remove(nodeAttrIndexIncludeRoutingSetting); + } mb.put(IndexMetadata.builder(indexMetadata) .settings(finalSettings) @@ -444,6 +456,23 @@ private static Settings maybeMigrateRoutingSettingToTierPreference(String attrib return newSettingsBuilder.build(); } + private static Settings migrateToDefaultTierPreference(ClusterState currentState, IndexMetadata indexMetadata) { + Settings currentIndexSettings = indexMetadata.getSettings(); + List tierPreference = DataTier.parseTierList(currentIndexSettings.get(DataTier.TIER_PREFERENCE)); + if (tierPreference.isEmpty() == false) { + return currentIndexSettings; + } + + Settings.Builder newSettingsBuilder = Settings.builder().put(currentIndexSettings); + String indexName = indexMetadata.getIndex().getName(); + + boolean isDataStream = currentState.metadata().findDataStreams(indexName).isEmpty() == false; + String convertedTierPreference = isDataStream ? DataTier.DATA_HOT : DataTier.DATA_CONTENT; + newSettingsBuilder.put(TIER_PREFERENCE, convertedTierPreference); + logger.debug("index [{}]: configured setting [{}] to [{}]", indexName, TIER_PREFERENCE, convertedTierPreference); + return newSettingsBuilder.build(); + } + /** * Converts the provided node attribute value to the corresponding `_tier_preference` configuration. * Known (and convertible) attribute values are: diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java index 1c757af351ffd..846bdadffaec2 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java @@ -56,6 +56,7 @@ import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.migrateToDataTiersRouting; import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -444,7 +445,7 @@ public void testMigrateIndices() { } { - // index with an unknown `data` attribute routing value should **not** be migrated + // index with an unknown `data` attribute routing value should be migrated IndexMetadata.Builder indexWithUnknownDataAttribute = IndexMetadata.builder("indexWithUnknownDataAttribute").settings(getBaseIndexSettings() .put(DATA_ROUTING_REQUIRE_SETTING, "something_else")); @@ -453,11 +454,12 @@ public void testMigrateIndices() { Metadata.Builder mb = Metadata.builder(state.metadata()); List migratedIndices = migrateIndices(mb, state, "data"); - assertThat(migratedIndices.size(), is(0)); + assertThat(migratedIndices.size(), is(1)); ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build(); IndexMetadata migratedIndex = migratedState.metadata().index("indexWithUnknownDataAttribute"); assertThat(migratedIndex.getSettings().get(DATA_ROUTING_REQUIRE_SETTING), is("something_else")); + assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_content")); } { @@ -483,7 +485,7 @@ public void testMigrateIndices() { } { - // index that doesn't have any data attribute routing but has another attribute should not see any change + // index that doesn't have any data attribute routing but has another attribute should be migrated IndexMetadata.Builder indexBoxAttribute = IndexMetadata.builder("indexWithBoxAttribute").settings(getBaseIndexSettings() .put(BOX_ROUTING_REQUIRE_SETTING, "warm")); @@ -492,13 +494,13 @@ public void testMigrateIndices() { Metadata.Builder mb = Metadata.builder(state.metadata()); List migratedIndices = migrateIndices(mb, state, "data"); - assertThat(migratedIndices.size(), is(0)); + assertThat(migratedIndices.size(), is(1)); ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build(); IndexMetadata migratedIndex = migratedState.metadata().index("indexWithBoxAttribute"); assertThat(migratedIndex.getSettings().get(DATA_ROUTING_REQUIRE_SETTING), nullValue()); assertThat(migratedIndex.getSettings().get(BOX_ROUTING_REQUIRE_SETTING), is("warm")); - assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), nullValue()); + assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_content")); } { @@ -509,13 +511,13 @@ public void testMigrateIndices() { Metadata.Builder mb = Metadata.builder(state.metadata()); List migratedIndices = migrateIndices(mb, state, "data"); - assertThat(migratedIndices.size(), is(0)); + assertThat(migratedIndices.size(), is(1)); ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build(); IndexMetadata migratedIndex = migratedState.metadata().index("indexNoRoutingAttribute"); assertThat(migratedIndex.getSettings().get(DATA_ROUTING_REQUIRE_SETTING), nullValue()); assertThat(migratedIndex.getSettings().get(BOX_ROUTING_REQUIRE_SETTING), nullValue()); - assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), nullValue()); + assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_content")); } } @@ -592,8 +594,8 @@ public void testMigrateToDataTiersRouting() { assertThat(migratedEntities.removedIndexTemplateName, is("catch-all")); assertThat(migratedEntities.migratedPolicies.size(), is(1)); assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); - assertThat(migratedEntities.migratedIndices.size(), is(1)); - assertThat(migratedEntities.migratedIndices.get(0), is("indexWithWarmDataAttribute")); + assertThat(migratedEntities.migratedIndices.size(), is(2)); + assertThat(migratedEntities.migratedIndices, hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); ClusterState newState = migratedEntitiesTuple.v1(); assertThat(newState.metadata().getTemplates().size(), is(1)); @@ -610,8 +612,8 @@ public void testMigrateToDataTiersRouting() { assertThat(migratedEntities.removedIndexTemplateName, nullValue()); assertThat(migratedEntities.migratedPolicies.size(), is(1)); assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); - assertThat(migratedEntities.migratedIndices.size(), is(1)); - assertThat(migratedEntities.migratedIndices.get(0), is("indexWithWarmDataAttribute")); + assertThat(migratedEntities.migratedIndices.size(), is(2)); + assertThat(migratedEntities.migratedIndices, hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); ClusterState newState = migratedEntitiesTuple.v1(); assertThat(newState.metadata().getTemplates().size(), is(2)); @@ -627,11 +629,14 @@ public void testMigrateToDataTiersRouting() { MigratedEntities migratedEntities = migratedEntitiesTuple.v2(); assertThat(migratedEntities.migratedPolicies.size(), is(1)); assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); - assertThat(migratedEntities.migratedIndices.size(), is(1)); - assertThat(migratedEntities.migratedIndices.get(0), is("indexWithWarmDataAttribute")); + assertThat(migratedEntities.migratedIndices.size(), is(2)); + assertThat(migratedEntities.migratedIndices, hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); - IndexMetadata migratedIndex = migratedEntitiesTuple.v1().metadata().index("indexWithWarmDataAttribute"); + IndexMetadata migratedIndex; + migratedIndex = migratedEntitiesTuple.v1().metadata().index("indexWithWarmDataAttribute"); assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_warm,data_hot")); + migratedIndex = migratedEntitiesTuple.v1().metadata().index("indexWithUnknownDataAttribute"); + assertThat(migratedIndex.getSettings().get(TIER_PREFERENCE), is("data_content")); } } From bde16e990639a36913fda464be2a6608807eda5f Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 17:26:20 -0400 Subject: [PATCH 08/12] Switch these casts --- .../org/elasticsearch/xpack/MigrateToDataTiersIT.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index ae3f3cb0d3d62..36a37aecb6f50 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -36,8 +36,8 @@ import org.junit.Before; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -163,9 +163,9 @@ public void testMigrateToDataTiersAction() throws Exception { assertOK(migrateDeploymentResponse); Map migrateResponseAsMap = responseAsMap(migrateDeploymentResponse); - assertThat((ArrayList) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()), + assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()), containsInAnyOrder(policy)); - assertThat((ArrayList) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()), + assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()), containsInAnyOrder(index, indexWithDataWarmRouting)); assertThat(migrateResponseAsMap.get(MigrateToDataTiersResponse.REMOVED_LEGACY_TEMPLATE.getPreferredName()), is(templateName)); @@ -270,9 +270,9 @@ public void testMigrationDryRun() throws Exception { // response should contain the correct "to migrate" entities Map migrateResponseAsMap = responseAsMap(migrateDeploymentResponse); - assertThat((ArrayList) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()), + assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()), containsInAnyOrder(policy)); - assertThat((ArrayList) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()), + assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()), containsInAnyOrder(index, indexWithDataWarmRouting)); assertThat(migrateResponseAsMap.get(MigrateToDataTiersResponse.REMOVED_LEGACY_TEMPLATE.getPreferredName()), is(templateName)); From df9e3fcf1cf3a430f176d72b7e9d7d918d05bbdc Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 17:28:27 -0400 Subject: [PATCH 09/12] Drop the randomIntBetween here I don't think it's buying us anything --- .../java/org/elasticsearch/xpack/MigrateToDataTiersIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index 36a37aecb6f50..4863a88c746a1 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -130,7 +130,7 @@ public void testMigrateToDataTiersAction() throws Exception { createNewSingletonPolicy(client(), rolloverOnlyPolicyName, "hot", new RolloverAction(null, null, null, 1L)); String rolloverIndexPrefix = "rolloverpolicytest_index"; - for (int i = 1; i < randomIntBetween(2, 5); i++) { + for (int i = 1; i <= 2; i++) { // assign the rollover-only policy to a few other indices - these indices and the rollover-only policy should not be migrated // in any way createIndexWithSettings(client(), rolloverIndexPrefix + "-00000" + i, alias + i, Settings.builder() From 4dbca823f35fa77e965c387cfd2f526a892374b7 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 17:29:30 -0400 Subject: [PATCH 10/12] contains is a little better here It's not like there's an order to care about anyway --- .../java/org/elasticsearch/xpack/MigrateToDataTiersIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index 4863a88c746a1..1f6eb88b4163b 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -49,6 +49,7 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -164,7 +165,7 @@ public void testMigrateToDataTiersAction() throws Exception { Map migrateResponseAsMap = responseAsMap(migrateDeploymentResponse); assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()), - containsInAnyOrder(policy)); + contains(policy)); assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()), containsInAnyOrder(index, indexWithDataWarmRouting)); assertThat(migrateResponseAsMap.get(MigrateToDataTiersResponse.REMOVED_LEGACY_TEMPLATE.getPreferredName()), From 2f71872820c526150dde973e3ab6aed195ea3ea4 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 13 Oct 2021 17:30:04 -0400 Subject: [PATCH 11/12] Get this test passing --- .../java/org/elasticsearch/xpack/MigrateToDataTiersIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index 1f6eb88b4163b..0f6adf1b9b479 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -132,8 +132,8 @@ public void testMigrateToDataTiersAction() throws Exception { String rolloverIndexPrefix = "rolloverpolicytest_index"; for (int i = 1; i <= 2; i++) { - // assign the rollover-only policy to a few other indices - these indices and the rollover-only policy should not be migrated - // in any way + // assign the rollover-only policy to a few other indices - these indices will end up getting caught by the catch-all + // tier preference migration createIndexWithSettings(client(), rolloverIndexPrefix + "-00000" + i, alias + i, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) @@ -167,7 +167,7 @@ public void testMigrateToDataTiersAction() throws Exception { assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()), contains(policy)); assertThat((List) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()), - containsInAnyOrder(index, indexWithDataWarmRouting)); + containsInAnyOrder(index, indexWithDataWarmRouting, rolloverIndexPrefix + "-000001", rolloverIndexPrefix + "-000002")); assertThat(migrateResponseAsMap.get(MigrateToDataTiersResponse.REMOVED_LEGACY_TEMPLATE.getPreferredName()), is(templateName)); From 5d2088d087af332122297470811d3de69bab8670 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 15 Oct 2021 14:05:15 -0400 Subject: [PATCH 12/12] Set ENFORCE_DEFAULT_TIER_PREFERENCE when migrating --- .../xpack/MigrateToDataTiersIT.java | 10 ++++++++ ...adataMigrateToDataTiersRoutingService.java | 13 ++++++++++ ...MigrateToDataTiersRoutingServiceTests.java | 24 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index 0ad1b3807279f..2fe831273c36f 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; @@ -210,6 +213,13 @@ public void testMigrateToDataTiersAction() throws Exception { assertThat(cachedPhaseDefinition, containsString(ShrinkAction.NAME)); assertThat(cachedPhaseDefinition, containsString(SetPriorityAction.NAME)); assertThat(cachedPhaseDefinition, containsString(ForceMergeAction.NAME)); + + // ENFORCE_DEFAULT_TIER_PREFERENCE has been set to true + Request getSettingsRequest = new Request("GET", "_cluster/settings"); + Response getSettingsResponse = client().performRequest(getSettingsRequest); + ObjectMapper mapper = new ObjectMapper(); + JsonNode json = mapper.readTree(getSettingsResponse.getEntity().getContent()); + assertTrue(json.at("/persistent/cluster/routing/allocation/enforce_default_tier_preference").asBoolean()); } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index 687d4bb723bc8..9d91edde74e38 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -48,6 +48,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING; +import static org.elasticsearch.cluster.routing.allocation.DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE; import static org.elasticsearch.cluster.routing.allocation.DataTier.TIER_PREFERENCE; import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPED; @@ -127,6 +128,18 @@ public static Tuple migrateToDataTiersRouting(Cl } Metadata.Builder mb = Metadata.builder(currentState.metadata()); + + // set ENFORCE_DEFAULT_TIER_PREFERENCE to true (in the persistent settings) + mb.persistentSettings(Settings.builder() + .put(mb.persistentSettings()) + .put(ENFORCE_DEFAULT_TIER_PREFERENCE, true) + .build()); + + // and remove it from the transient settings, just in case it was there + Settings.Builder transientSettingsBuilder = Settings.builder().put(mb.transientSettings()); + transientSettingsBuilder.remove(ENFORCE_DEFAULT_TIER_PREFERENCE); + mb.transientSettings(transientSettingsBuilder.build()); + String removedIndexTemplateName = null; if (Strings.hasText(indexTemplateToDelete)) { if (currentState.metadata().getTemplates().containsKey(indexTemplateToDelete)) { diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java index 846bdadffaec2..ceeab225d9ecc 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; @@ -48,6 +49,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING; +import static org.elasticsearch.cluster.routing.allocation.DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE; import static org.elasticsearch.cluster.routing.allocation.DataTier.TIER_PREFERENCE; import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.allocateActionDefinesRoutingRules; import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.convertAttributeValueToTierPreference; @@ -690,6 +692,28 @@ public void testMigrationDoesNotRemoveComposableTemplates() { assertThat(migratedEntitiesTuple.v1().metadata().templatesV2().get(composableTemplateName), is(composableIndexTemplate)); } + public void testMigrationSetsEnforceTierPreferenceToTrue() { + ClusterState clusterState; + Tuple migratedEntitiesTuple; + Metadata.Builder metadata; + + // if the cluster state doesn't mention the setting, it ends up true + clusterState = ClusterState.builder(ClusterName.DEFAULT).build(); + migratedEntitiesTuple = migrateToDataTiersRouting(clusterState, null, null, REGISTRY, client, null); + assertTrue(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.get(migratedEntitiesTuple.v1().metadata().persistentSettings())); + assertFalse(migratedEntitiesTuple.v1().metadata().transientSettings().keySet().contains(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE)); + + // regardless of the true/false combinations of persistent/transient settings for ENFORCE_DEFAULT_TIER_PREFERENCE, + // it ends up set to true as a persistent setting (and if there was a transient setting, it was removed) + metadata = Metadata.builder(); + metadata.persistentSettings(Settings.builder().put(ENFORCE_DEFAULT_TIER_PREFERENCE, randomBoolean()).build()); + metadata.transientSettings(Settings.builder().put(ENFORCE_DEFAULT_TIER_PREFERENCE, randomBoolean()).build()); + clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + migratedEntitiesTuple = migrateToDataTiersRouting(clusterState, null, null, REGISTRY, client, null); + assertTrue(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.get(migratedEntitiesTuple.v1().metadata().persistentSettings())); + assertFalse(migratedEntitiesTuple.v1().metadata().transientSettings().keySet().contains(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE)); + } + private LifecyclePolicyMetadata getWarmColdPolicyMeta(SetPriorityAction setPriorityAction, ShrinkAction shrinkAction, AllocateAction warmAllocateAction, AllocateAction coldAllocateAction) { LifecyclePolicy policy = new LifecyclePolicy(lifecycleName,