From 679bedaf622e72f446a9b2737986b0ecca8945a2 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Tue, 19 Oct 2021 14:17:10 -0400 Subject: [PATCH] Add deprecation info and warnings for an empty TIER_PREFERENCE (#79305) --- .../metadata/MetadataCreateIndexService.java | 14 ++- .../cluster/routing/allocation/DataTier.java | 24 ++++ .../MetadataCreateIndexServiceTests.java | 14 ++- .../routing/allocation/DataTierTests.java | 43 +++++++ .../allocation/FailedNodeRoutingTests.java | 3 +- ...ClusterStateServiceRandomUpdatesTests.java | 4 +- .../xpack/deprecation/DeprecationChecks.java | 38 +++--- .../deprecation/DeprecationInfoAction.java | 23 ++-- .../deprecation/IndexDeprecationChecks.java | 25 +++- .../DeprecationInfoActionResponseTests.java | 13 +- .../IndexDeprecationChecksTests.java | 119 ++++++++++++++---- 11 files changed, 253 insertions(+), 67 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index da837b7c62694..53e8bd84eaff0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -814,16 +814,26 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu indexSettingsBuilder.put(requestSettings.build()); if (sourceMetadata == null) { // not for shrink/split/clone + String currentTierPreference = indexSettingsBuilder.get(DataTier.TIER_PREFERENCE); + List preferredTiers = DataTier.parseTierList(currentTierPreference); if (enforceDefaultTierPreference) { // regardless of any previous logic, we're going to force there // to be an appropriate non-empty value for the tier preference - String currentTierPreference = indexSettingsBuilder.get(DataTier.TIER_PREFERENCE); - if (DataTier.parseTierList(currentTierPreference).isEmpty()) { + if (preferredTiers.isEmpty()) { String newTierPreference = isDataStreamIndex ? DataTier.DATA_HOT : DataTier.DATA_CONTENT; logger.debug("enforcing default [{}] setting for [{}] creation, replacing [{}] with [{}]", DataTier.TIER_PREFERENCE, request.index(), currentTierPreference, newTierPreference); indexSettingsBuilder.put(DataTier.TIER_PREFERENCE, newTierPreference); } + } else { + // if we're not enforcing the tier preference, then maybe warn + if (preferredTiers.isEmpty()) { + if (DataTier.dataNodesWithoutAllDataRoles(currentState).isEmpty() == false) { + DEPRECATION_LOGGER.warn(DeprecationCategory.INDICES, "index_without_tier_preference", + "[{}] creating index with an empty [{}] setting, in 8.0 this setting will be required for all indices", + request.index(), DataTier.TIER_PREFERENCE); + } + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java index 98a0b36e8bd1d..ae9b1ff21a17e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; @@ -28,6 +29,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; /** * The {@code DataTier} class encapsulates the formalization of the "content", @@ -266,4 +268,26 @@ public Iterator> settings() { return dependencies.iterator(); } } + + /** + * Checks each data node in the cluster state to see whether it has the explicit data role or if it has + * all data tiers (e.g. 'data_hot', 'data_warm', etc). The former condition being treated as a shortcut + * for the latter condition (see DataTierAllocationDecider#allocationAllowed(String, Set)) for details, + * as well as the various DataTier#isFooNode(DiscoveryNode) methods. + * + * @param clusterState the cluster state + * @return a set of data nodes that do not have all data roles + */ + public static Set dataNodesWithoutAllDataRoles(ClusterState clusterState) { + return clusterState.getNodes().getDataNodes().values().stream() + .filter(node -> { + Set roles = node.getRoles().stream() + .map(DiscoveryNodeRole::roleName) + .collect(Collectors.toSet()); + + return roles.contains(DiscoveryNodeRole.DATA_ROLE.roleName()) == false && + roles.containsAll(ALL_DATA_TIERS) == false; + }) + .collect(Collectors.toSet()); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index c1396f94df350..294598b3de1f4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.cluster.routing.allocation.DataTierTests; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; @@ -996,13 +997,24 @@ private Optional aggregatedTierPreference(Settings settings, boolean isD } else { request.dataStreamName(null); } - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateSettings, + + ClusterState clusterState = ClusterState.EMPTY_STATE; + if (randomBoolean()) { + clusterState = DataTierTests.clusterStateWithoutAllDataRoles(); + } + + Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateSettings, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), org.elasticsearch.core.Set.of(new DataTier.DefaultHotAllocationSettingProvider()), enforceDefaultTierPreference); if (aggregatedIndexSettings.keySet().contains(DataTier.TIER_PREFERENCE)) { return Optional.of(aggregatedIndexSettings.get(DataTier.TIER_PREFERENCE)); } else { + if (clusterState != ClusterState.EMPTY_STATE) { + assertWarnings("[" + request.index() + "] creating index with " + + "an empty [" + DataTier.TIER_PREFERENCE + "] setting, in 8.0 this setting will be required for all indices"); + } + return Optional.empty(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java index 2a9b2679732a9..e22933469d2bc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.routing.allocation; import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -21,20 +22,27 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import static org.elasticsearch.cluster.routing.allocation.DataTier.ALL_DATA_TIERS; import static org.elasticsearch.cluster.routing.allocation.DataTier.DATA_COLD; import static org.elasticsearch.cluster.routing.allocation.DataTier.DATA_HOT; import static org.elasticsearch.cluster.routing.allocation.DataTier.DATA_WARM; import static org.elasticsearch.cluster.routing.allocation.DataTier.getPreferredTiersConfiguration; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; public class DataTierTests extends ESTestCase { @@ -139,6 +147,41 @@ public void testGetPreferredTiersConfiguration() { assertThat(exception.getMessage(), is("invalid data tier [no_tier]")); } + public void testDataNodesWithoutAllDataRoles() { + ClusterState clusterState = clusterStateWithoutAllDataRoles(); + Set nodes = DataTier.dataNodesWithoutAllDataRoles(clusterState); + assertThat(nodes, hasSize(2)); + assertThat(nodes, hasItem(both(hasProperty("name", is("name_3"))) + .and(hasProperty("roles", contains(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE))))); + assertThat(nodes, hasItem(hasProperty("name", is("name_4")))); + } + + public static ClusterState clusterStateWithoutAllDataRoles() { + Set allDataRoles = new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES).stream() + .filter(role -> ALL_DATA_TIERS.contains(role.roleName())).collect(Collectors.toSet()); + + Collection allButOneDataTiers = randomValueOtherThan(ALL_DATA_TIERS, + () -> randomSubsetOf(randomIntBetween(1, ALL_DATA_TIERS.size() - 1), ALL_DATA_TIERS)); + Set allButOneDataRoles = new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES).stream() + .filter(role -> allButOneDataTiers.contains(role.roleName())).collect(Collectors.toSet()); + + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + List nodesList = org.elasticsearch.core.List.of( + newNode(0, org.elasticsearch.core.Map.of(), org.elasticsearch.core.Set.of(DiscoveryNodeRole.MASTER_ROLE)), + newNode(1, org.elasticsearch.core.Map.of(), org.elasticsearch.core.Set.of(DiscoveryNodeRole.DATA_ROLE)), + newNode(2, org.elasticsearch.core.Map.of(), allDataRoles), + newNode(3, org.elasticsearch.core.Map.of(), org.elasticsearch.core.Set.of(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)), + newNode(4, org.elasticsearch.core.Map.of(), allButOneDataRoles) + ); + for (DiscoveryNode node : nodesList) { + discoBuilder = discoBuilder.add(node); + } + discoBuilder.localNodeId(randomFrom(nodesList).getId()); + discoBuilder.masterNodeId(randomFrom(nodesList).getId()); + + return ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoBuilder.build()).build(); + } + private static DiscoveryNodes buildDiscoveryNodes() { int numNodes = randomIntBetween(3, 15); DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java index 6deca989932bb..74e5483509728 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java @@ -132,7 +132,8 @@ public void testRandomClusterPromotesNewestReplica() throws InterruptedException String name = "index_" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT); Settings.Builder settingsBuilder = Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 4)) - .put(SETTING_NUMBER_OF_REPLICAS, randomIntBetween(2, 4)); + .put(SETTING_NUMBER_OF_REPLICAS, randomIntBetween(2, 4)) + .put(DataTier.TIER_PREFERENCE, DataTier.DATA_CONTENT); CreateIndexRequest request = new CreateIndexRequest(name, settingsBuilder.build()).waitForActiveShards(ActiveShardCount.NONE); state = cluster.createIndex(state, request); assertTrue(state.metadata().hasIndex(name)); diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index c302bcc6606f1..06504e027aa6b 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.FailedShard; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; @@ -341,7 +342,8 @@ public ClusterState randomlyUpdateClusterState(ClusterState state, } String name = "index_" + randomAlphaOfLength(15).toLowerCase(Locale.ROOT); Settings.Builder settingsBuilder = Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3)); + .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3)) + .put(DataTier.TIER_PREFERENCE, DataTier.DATA_CONTENT); if (randomBoolean()) { int min = randomInt(2); int max = min + randomInt(3); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 28a219006d102..26c32a86d0317 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -13,14 +13,15 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -135,24 +136,25 @@ private DeprecationChecks() { ).collect(Collectors.toList()); } - static List> INDEX_SETTINGS_CHECKS = + static List> INDEX_SETTINGS_CHECKS = Collections.unmodifiableList(Arrays.asList( - IndexDeprecationChecks::oldIndicesCheck, - IndexDeprecationChecks::tooManyFieldsCheck, - IndexDeprecationChecks::chainedMultiFieldsCheck, - IndexDeprecationChecks::deprecatedDateTimeFormat, - IndexDeprecationChecks::translogRetentionSettingCheck, - IndexDeprecationChecks::fieldNamesDisabledCheck, - IndexDeprecationChecks::checkIndexDataPath, - IndexDeprecationChecks::indexingSlowLogLevelSettingCheck, - IndexDeprecationChecks::searchSlowLogLevelSettingCheck, - IndexDeprecationChecks::storeTypeSettingCheck, - IndexDeprecationChecks::checkIndexRoutingRequireSetting, - IndexDeprecationChecks::checkIndexRoutingIncludeSetting, - IndexDeprecationChecks::checkIndexRoutingExcludeSetting, - IndexDeprecationChecks::checkIndexMatrixFiltersSetting, - IndexDeprecationChecks::checkGeoShapeMappings, - IndexDeprecationChecks::frozenIndexSettingCheck + (clusterState, indexMetadata) -> IndexDeprecationChecks.oldIndicesCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.tooManyFieldsCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.chainedMultiFieldsCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.deprecatedDateTimeFormat(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.translogRetentionSettingCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.fieldNamesDisabledCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.checkIndexDataPath(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.indexingSlowLogLevelSettingCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.searchSlowLogLevelSettingCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.storeTypeSettingCheck(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.checkIndexRoutingRequireSetting(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.checkIndexRoutingIncludeSetting(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.checkIndexRoutingExcludeSetting(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.checkIndexMatrixFiltersSetting(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.checkGeoShapeMappings(indexMetadata), + (clusterState, indexMetadata) -> IndexDeprecationChecks.frozenIndexSettingCheck(indexMetadata), + IndexDeprecationChecks::emptyDataTierPreferenceCheck )); /** diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java index cf36313bbd863..e9c6410800aa8 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java @@ -25,9 +25,9 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.io.IOException; @@ -39,6 +39,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -198,14 +199,16 @@ public int hashCode() { * @param clusterSettingsChecks The list of cluster-level checks * @return The list of deprecation issues found in the cluster */ - public static DeprecationInfoAction.Response from(ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - Request request, - NodesDeprecationCheckResponse nodeDeprecationResponse, - List> indexSettingsChecks, - List> clusterSettingsChecks, - Map> pluginSettingIssues, - List skipTheseDeprecatedSettings) { + public static DeprecationInfoAction.Response from( + ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver, + Request request, + NodesDeprecationCheckResponse nodeDeprecationResponse, + List> indexSettingsChecks, + List> clusterSettingsChecks, + Map> pluginSettingIssues, + List skipTheseDeprecatedSettings + ) { // Allow system index access here to prevent deprecation warnings when we call this API String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(state, request); ClusterState stateWithSkippedSettingsRemoved = removeSkippedSettings(state, concreteIndexNames, skipTheseDeprecatedSettings); @@ -217,7 +220,7 @@ public static DeprecationInfoAction.Response from(ClusterState state, for (String concreteIndex : concreteIndexNames) { IndexMetadata indexMetadata = stateWithSkippedSettingsRemoved.getMetadata().index(concreteIndex); List singleIndexIssues = filterChecks(indexSettingsChecks, - c -> c.apply(indexMetadata)); + c -> c.apply(state, indexMetadata)); if (singleIndexIssues.size() > 0) { indexSettingsIssues.put(concreteIndex, singleIndexIssues); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java index 6c64c71256cd6..73e5b02d11dc5 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java @@ -8,20 +8,22 @@ import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.joda.JodaDeprecationPatterns; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexingSlowLog; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; -import org.elasticsearch.index.engine.frozen.FrozenEngine; -import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.index.SearchSlowLog; import org.elasticsearch.index.SlowLogLevel; +import org.elasticsearch.index.engine.frozen.FrozenEngine; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.util.ArrayList; import java.util.Collections; @@ -438,4 +440,21 @@ static DeprecationIssue frozenIndexSettingCheck(IndexMetadata indexMetadata) { } return null; } + + static DeprecationIssue emptyDataTierPreferenceCheck(ClusterState clusterState, IndexMetadata indexMetadata) { + if (DataTier.dataNodesWithoutAllDataRoles(clusterState).isEmpty() == false) { + final List tierPreference = DataTier.parseTierList(DataTier.TIER_PREFERENCE_SETTING.get(indexMetadata.getSettings())); + if (tierPreference.isEmpty()) { + String indexName = indexMetadata.getIndex().getName(); + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "index [" + indexName + "] does not have a [" + DataTier.TIER_PREFERENCE + "] setting, " + + "in 8.0 this setting will be required for all indices and may not be empty or null.", + "https://www.elastic.co/guide/en/elasticsearch/reference/current/data-tiers.html", + "Update the settings for this index to specify an appropriate tier preference.", + false, + null); + } + } + return null; + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java index 14ec078f9cdfc..adc365b301c8e 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java @@ -18,11 +18,11 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.core.Tuple; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; import org.junit.Assert; @@ -36,6 +36,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -97,9 +98,9 @@ public void testFrom() throws IOException { Collections.unmodifiableList(Arrays.asList( (s) -> clusterIssueFound ? foundIssue : null )); - List> indexSettingsChecks = + List> indexSettingsChecks = Collections.unmodifiableList(Arrays.asList( - (idx) -> indexIssueFound ? foundIssue : null + (cs, idx) -> indexIssueFound ? foundIssue : null )); NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( @@ -170,9 +171,9 @@ public void testRemoveSkippedSettings() throws IOException { } )); AtomicReference visibleIndexSettings = new AtomicReference<>(); - List> indexSettingsChecks = + List> indexSettingsChecks = Collections.unmodifiableList(Arrays.asList( - (idx) -> { + (cs, idx) -> { visibleIndexSettings.set(idx.getSettings()); return null; } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java index 349aab69303bb..f51bef56e302e 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java @@ -8,24 +8,28 @@ package org.elasticsearch.xpack.deprecation; import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.joda.JodaDeprecationPatterns; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexingSlowLog; import org.elasticsearch.index.SearchSlowLog; import org.elasticsearch.index.SlowLogLevel; -import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.engine.frozen.FrozenEngine; +import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.io.IOException; @@ -63,7 +67,8 @@ public void testOldIndicesCheck() { "Index created before 7.0", "https://ela.st/es-deprecation-7-reindex", "This index was created using version: " + createdWith, false, null); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); assertEquals(singletonList(expected), issues); } @@ -88,7 +93,8 @@ public void testTooManyFieldsCheck() throws IOException { .numberOfReplicas(randomIntBetween(1, 100)) .putMapping("_doc", simpleMapping) .build(); - List noIssues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List noIssues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertEquals(0, noIssues.size()); // Test that it catches having too many fields @@ -118,7 +124,8 @@ public void testTooManyFieldsCheck() throws IOException { "and does not have [" + IndexSettings.DEFAULT_FIELD_SETTING.getKey() + "] set, which may cause queries which use " + "automatic field expansion, such as query_string, simple_query_string, and multi_match to fail if fields are not " + "explicitly specified in the query.", false, null); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(tooManyFieldsIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, tooManyFieldsIndex)); assertEquals(singletonList(expected), issues); // Check that it's okay to have too many fields as long as `index.query.default_field` is set @@ -130,7 +137,8 @@ public void testTooManyFieldsCheck() throws IOException { .putMapping("_doc", Strings.toString(mappingBuilder)) .build(); List withDefaultFieldIssues = - DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(tooManyFieldsOk)); + DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, tooManyFieldsOk)); assertEquals(0, withDefaultFieldIssues.size()); } @@ -168,7 +176,8 @@ public void testChainedMultiFields() throws IOException { .numberOfReplicas(1) .putMapping("_doc", mapping) .build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertEquals(1, issues.size()); DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING, @@ -228,7 +237,8 @@ public void testMultipleWarningsOnCombinedPattern() throws IOException { "; "+ "'Y' year-of-era should be replaced with 'y'. Use 'Y' for week-based-year.]"+ "]. "+ JodaDeprecationPatterns.USE_NEW_FORMAT_SPECIFIERS, false, null); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertThat(issues, hasItem(expected)); } @@ -250,7 +260,8 @@ public void testDuplicateWarningsOnCombinedPattern() throws IOException { "[type: _doc, field: date_time_field_Y, format: dd-YYYY||MM-YYYY, " + "suggestion: 'Y' year-of-era should be replaced with 'y'. Use 'Y' for week-based-year.]"+ "]. "+ JodaDeprecationPatterns.USE_NEW_FORMAT_SPECIFIERS, false, null); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertThat(issues, hasItem(expected)); } @@ -272,7 +283,8 @@ public void testWarningsOnMixCustomAndDefinedPattern() throws IOException { "[type: _doc, field: date_time_field_Y, format: strictWeekyearWeek||MM-YYYY, " + "suggestion: 'Y' year-of-era should be replaced with 'y'. Use 'Y' for week-based-year.]"+ "]. "+ JodaDeprecationPatterns.USE_NEW_FORMAT_SPECIFIERS, false, null); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertThat(issues, hasItem(expected)); } @@ -326,7 +338,8 @@ public void testJodaPatternDeprecations() throws IOException { "suggestion: 'z' time zone text. Will print 'Z' for Zulu given UTC timezone." + "]"+ "]. "+ JodaDeprecationPatterns.USE_NEW_FORMAT_SPECIFIERS, false, null); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertThat(issues, hasItem(expected)); } @@ -353,7 +366,8 @@ public void testMultipleJodaPatternDeprecationInOneField() throws IOException { "'x' weak-year should be replaced with 'Y'. Use 'x' for zone-offset." + "]"+ "]. "+ JodaDeprecationPatterns.USE_NEW_FORMAT_SPECIFIERS, false, null); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertThat(issues, hasItem(expected)); } @@ -408,7 +422,8 @@ public void testTranslogRetentionSettings() { settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b"); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); assertThat(issues, contains( new DeprecationIssue(DeprecationIssue.Level.WARNING, "translog retention settings are ignored", @@ -427,7 +442,8 @@ public void testDefaultTranslogRetentionSettings() { settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false); } IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); assertThat(issues, empty()); } @@ -445,7 +461,8 @@ public void testFieldNamesEnabling() throws IOException { .numberOfShards(1) .numberOfReplicas(0) .putMapping("_doc", mapping).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, simpleIndex)); assertEquals(1, issues.size()); DeprecationIssue issue = issues.get(0); @@ -460,7 +477,8 @@ public void testIndexDataPathSetting() { Settings.Builder settings = settings(Version.CURRENT); settings.put(IndexMetadata.INDEX_DATA_PATH_SETTING.getKey(), createTempDir()); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); final String expectedUrl = "https://ela.st/es-deprecation-7-shared-path-settings"; assertThat(issues, contains( new DeprecationIssue(DeprecationIssue.Level.CRITICAL, @@ -475,7 +493,8 @@ public void testSlowLogLevel() { settings.put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL.getKey(), SlowLogLevel.DEBUG); settings.put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_LEVEL_SETTING.getKey(), SlowLogLevel.DEBUG); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); final String expectedUrl = "https://ela.st/es-deprecation-7-slowlog-settings"; assertThat(issues, containsInAnyOrder( new DeprecationIssue(DeprecationIssue.Level.WARNING, @@ -494,7 +513,8 @@ public void testSimpleFSSetting() { Settings.Builder settings = settings(Version.CURRENT); settings.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "simplefs"); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); assertThat(issues, contains( new DeprecationIssue(DeprecationIssue.Level.WARNING, "[simplefs] is deprecated and will be removed in future versions", @@ -577,7 +597,8 @@ public void testCheckGeoShapeMappings() throws Exception { Settings.Builder settings = settings(Version.CURRENT); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); assertTrue(issues.isEmpty()); Map okGeoMappingMap = Collections.singletonMap("properties", Collections.singletonMap("location", @@ -585,7 +606,8 @@ public void testCheckGeoShapeMappings() throws Exception { mappingMetadata = new MappingMetadata("", okGeoMappingMap); IndexMetadata indexMetadata2 = IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); - issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata2)); + issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata2)); assertTrue(issues.isEmpty()); Map deprecatedPropertiesMap = Stream.of(new String[][] { @@ -598,7 +620,8 @@ public void testCheckGeoShapeMappings() throws Exception { mappingMetadata = new MappingMetadata("", deprecatedGeoMappingMap); IndexMetadata indexMetadata3 = IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); - issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata3)); + issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata3)); assertEquals(1, issues.size()); assertThat(issues, contains( new DeprecationIssue(DeprecationIssue.Level.CRITICAL, @@ -617,7 +640,8 @@ public void testCheckGeoShapeMappings() throws Exception { mappingMetadata = new MappingMetadata("", nestedDeprecatedGeoMappingMap); IndexMetadata indexMetadata4 = IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); - issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata4)); + issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata4)); assertEquals(1, issues.size()); assertThat(issues, contains( new DeprecationIssue(DeprecationIssue.Level.CRITICAL, @@ -632,7 +656,8 @@ public void testAdjacencyMatrixSetting() { Settings.Builder settings = settings(Version.CURRENT); settings.put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), 5); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); assertThat( issues, contains( @@ -660,7 +685,8 @@ public void testFrozenIndex() { Settings.Builder settings = settings(Version.CURRENT); settings.put(FrozenEngine.INDEX_FROZEN.getKey(), true); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c + -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); assertThat(issues, contains( new DeprecationIssue(DeprecationIssue.Level.WARNING, "index [test] is a frozen index. The frozen indices feature is deprecated and will be removed in a future version", @@ -671,4 +697,47 @@ public void testFrozenIndex() { ) )); } + + public void testEmptyDataTierPreference() { + Settings.Builder settings = settings(Version.CURRENT); + settings.put(DataTier.TIER_PREFERENCE_SETTING.getKey(), " "); + IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings) + .numberOfShards(randomIntBetween(1, 100)) + .numberOfReplicas(randomIntBetween(1, 100)) + .build(); + + { + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)); + assertThat(issues, empty()); + } + + { + ClusterState clusterState = clusterStateWithoutAllDataRoles(); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, + c -> c.apply(clusterState, indexMetadata)); + assertThat(issues, contains( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "index [test] does not have a [index.routing.allocation.include._tier_preference] setting, " + + "in 8.0 this setting will be required for all indices and may not be empty or null.", + "https://www.elastic.co/guide/en/elasticsearch/reference/current/data-tiers.html", + "Update the settings for this index to specify an appropriate tier preference.", false, null) + )); + } + } + + private static ClusterState clusterStateWithoutAllDataRoles() { + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + List nodesList = org.elasticsearch.core.List.of( + new DiscoveryNode("name_0", "node_0", buildNewFakeTransportAddress(), org.elasticsearch.core.Map.of(), + org.elasticsearch.core.Set.of(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE), Version.CURRENT) + ); + for (DiscoveryNode node : nodesList) { + discoBuilder = discoBuilder.add(node); + } + discoBuilder.localNodeId(randomFrom(nodesList).getId()); + discoBuilder.masterNodeId(randomFrom(nodesList).getId()); + + return ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoBuilder.build()).build(); + } }