diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index fd6fe61fd0c15..cd4d1bebc94be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; @@ -45,7 +46,6 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.PathUtils; @@ -66,6 +66,7 @@ import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -128,6 +129,8 @@ public class MetadataCreateIndexService { private final boolean forbidPrivateIndexSettings; private final Set indexSettingProviders = new HashSet<>(); + private volatile boolean enforceDefaultTierPreference; + public MetadataCreateIndexService( final Settings settings, final ClusterService clusterService, @@ -153,6 +156,14 @@ public MetadataCreateIndexService( this.systemIndices = systemIndices; this.forbidPrivateIndexSettings = forbidPrivateIndexSettings; this.shardLimitValidator = shardLimitValidator; + + enforceDefaultTierPreference = DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING, + this::setEnforceDefaultTierPreference); + } + + public void setEnforceDefaultTierPreference(boolean enforceDefaultTierPreference) { + this.enforceDefaultTierPreference = enforceDefaultTierPreference; } /** @@ -481,7 +492,8 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, resolveSettings(templates), - null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); + null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders, + this.enforceDefaultTierPreference); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -516,7 +528,8 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, resolveSettings(currentState.metadata(), templateName), - null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); + null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders, + this.enforceDefaultTierPreference); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -570,7 +583,8 @@ private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterSta settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders + indexSettingProviders, + this.enforceDefaultTierPreference ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -634,7 +648,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt } final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, Settings.EMPTY, - sourceMetadata, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); + sourceMetadata, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders, this.enforceDefaultTierPreference); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -696,7 +710,9 @@ static Map parseV1Mappings(String mappingsJson, List indexSettingProviders) { + Set indexSettingProviders, boolean enforceDefaultTierPreference) { + final boolean isDataStreamIndex = request.dataStreamName() != null; + // Create builders for the template and request settings. We transform these into builders // because we may want settings to be "removed" from these prior to being set on the new // index (see more comments below) @@ -711,7 +727,6 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu .put(request.settings()) .build(); - final boolean isDataStreamIndex = request.dataStreamName() != null; // Loop through all the explicit index setting providers, adding them to the // additionalIndexSettings map for (IndexSettingProvider provider : indexSettingProviders) { @@ -754,6 +769,20 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu // now, put the request settings, so they override templates indexSettingsBuilder.put(requestSettings.build()); + if (sourceMetadata == null) { // not for shrink/split/clone + if (enforceDefaultTierPreference) { + // regardless of any previous logic, we're going to force there + // to be an appropriate non-empty value for the tier preference + String currentTierPreference = indexSettingsBuilder.get(DataTier.TIER_PREFERENCE); + if (DataTier.parseTierList(currentTierPreference).isEmpty()) { + String newTierPreference = isDataStreamIndex ? DataTier.DATA_HOT : DataTier.DATA_CONTENT; + logger.debug("enforcing default [{}] setting for [{}] creation, replacing [{}] with [{}]", + DataTier.TIER_PREFERENCE, request.index(), currentTierPreference, newTierPreference); + indexSettingsBuilder.put(DataTier.TIER_PREFERENCE, newTierPreference); + } + } + } + if (indexSettingsBuilder.get(IndexMetadata.SETTING_VERSION_CREATED) == null) { final DiscoveryNodes nodes = currentState.nodes(); final Version createdVersion = Version.min(Version.CURRENT, nodes.getSmallestNonClientNodeVersion()); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java index 9a1ab49f55748..03ae442f40203 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.shard.IndexSettingProvider; @@ -44,6 +45,13 @@ public class DataTier { public static final Set ALL_DATA_TIERS = Set.of(DATA_CONTENT, DATA_HOT, DATA_WARM, DATA_COLD, DATA_FROZEN); + // this setting is for migrating from 7.x (where a tier preference was not required, and did not necessarily + // have a default value), to 8.x (where a tier preference will be required, and a default value will be injected). + // it will be removed as a breaking change in some future version, likely 9.0. + public static final String ENFORCE_DEFAULT_TIER_PREFERENCE = "cluster.routing.allocation.enforce_default_tier_preference"; + public static final Setting ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING = + Setting.boolSetting(ENFORCE_DEFAULT_TIER_PREFERENCE, false, Property.Dynamic, Property.NodeScope); + public static final String TIER_PREFERENCE = "index.routing.allocation.include._tier_preference"; private static final Settings DATA_CONTENT_TIER_PREFERENCE_SETTINGS = Settings.builder().put(TIER_PREFERENCE, DATA_CONTENT).build(); @@ -57,8 +65,8 @@ public class DataTier { DataTierSettingValidator::getDefaultTierPreference, Function.identity(), new DataTierSettingValidator(), - Setting.Property.Dynamic, - Setting.Property.IndexScope + Property.Dynamic, + Property.IndexScope ); static { diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index f03b49cda379d..228b6515136c9 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; @@ -485,7 +486,9 @@ public void apply(Settings value, Settings current, Settings previous) { FsHealthService.REFRESH_INTERVAL_SETTING, FsHealthService.SLOW_PATH_LOGGING_THRESHOLD_SETTING, IndexingPressure.MAX_INDEXING_BYTES, - ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE_FROZEN); + ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE_FROZEN, + DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING + ); static List> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 07712373dade8..b5dcd023a2f9a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; @@ -34,10 +35,8 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; @@ -57,6 +56,8 @@ import org.elasticsearch.test.gateway.TestGatewayAllocator; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentFactory; import org.hamcrest.Matchers; import org.junit.Before; @@ -68,6 +69,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -91,7 +93,6 @@ import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.getIndexNumberOfRoutingShards; import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.parseV1Mappings; import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases; - import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.hamcrest.Matchers.endsWith; @@ -580,7 +581,7 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet()); + Collections.emptySet(), randomBoolean()); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("value2")); @@ -633,7 +634,7 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateMetadata.settings(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet()); + Collections.emptySet(), randomBoolean()); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); assertThat(aggregatedIndexSettings.get("key1"), equalTo("requestValue")); @@ -648,7 +649,7 @@ null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLi public void testDefaultSettings() { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet()); + Collections.emptySet(), randomBoolean()); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); } @@ -656,7 +657,7 @@ null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLi public void testSettingsFromClusterState() { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), Collections.emptySet()); + randomShardLimitService(), Collections.emptySet(), randomBoolean()); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); } @@ -681,7 +682,7 @@ public void testTemplateOrder() throws Exception { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, MetadataIndexTemplateService.resolveSettings(templates), null, Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), Collections.emptySet()); + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), Collections.emptySet(), randomBoolean()); List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), MetadataIndexTemplateService.resolveAliases(templates), Metadata.builder().build(), aliasValidator, xContentRegistry(), searchExecutionContext, @@ -739,7 +740,7 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), clusterState.metadata().index("sourceIndex"), Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), Collections.emptySet()); + randomShardLimitService(), Collections.emptySet(), randomBoolean()); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); assertThat(aggregatedIndexSettings.get("requestSetting"), is("requestValue")); @@ -942,13 +943,88 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } + private Optional aggregatedTierPreference(Settings settings, boolean isDataStream, + boolean enforceDefaultTierPreference) { + Settings templateSettings = Settings.EMPTY; + request.settings(Settings.EMPTY); + + if (randomBoolean()) { + templateSettings = settings; + } else { + request.settings(settings); + } + + if (isDataStream) { + request.dataStreamName(randomAlphaOfLength(10)); + } else { + request.dataStreamName(null); + } + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateSettings, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Set.of(new DataTier.DefaultHotAllocationSettingProvider()), enforceDefaultTierPreference); + + if (aggregatedIndexSettings.keySet().contains(DataTier.TIER_PREFERENCE)) { + return Optional.of(aggregatedIndexSettings.get(DataTier.TIER_PREFERENCE)); + } else { + return Optional.empty(); + } + } + + public void testEnforceDefaultTierPreference() { + Settings settings; + Optional tier; + + // empty settings gets the appropriate tier + settings = Settings.EMPTY; + tier = aggregatedTierPreference(settings, false, randomBoolean()); + assertEquals(DataTier.DATA_CONTENT, tier.get()); + + settings = Settings.EMPTY; + tier = aggregatedTierPreference(settings, true, randomBoolean()); + assertEquals(DataTier.DATA_HOT, tier.get()); + + // an explicit tier is respected + settings = Settings.builder().put(DataTier.TIER_PREFERENCE, DataTier.DATA_COLD).build(); + tier = aggregatedTierPreference(settings, randomBoolean(), randomBoolean()); + assertEquals(DataTier.DATA_COLD, tier.get()); + + // any of the INDEX_ROUTING_.*_GROUP_PREFIX settings still result in a default if + // we're enforcing + String includeRoutingSetting = randomFrom( + IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX, + IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX, + IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX) + "." + randomAlphaOfLength(10); + settings = Settings.builder() + .put(includeRoutingSetting, randomAlphaOfLength(10)) + .build(); + tier = aggregatedTierPreference(settings, false, true); + assertEquals(DataTier.DATA_CONTENT, tier.get()); + // (continued from above) but not if we aren't + tier = aggregatedTierPreference(settings, false, false); + assertTrue(tier.isEmpty()); + + // an explicit null gets an empty tier if we're not enforcing + settings = Settings.builder().putNull(DataTier.TIER_PREFERENCE).build(); + tier = aggregatedTierPreference(settings, randomBoolean(), false); + assertTrue(tier.isEmpty()); + + // an explicit null gets the appropriate tier if we are enforcing + settings = Settings.builder().putNull(DataTier.TIER_PREFERENCE).build(); + tier = aggregatedTierPreference(settings, false, true); + assertEquals(DataTier.DATA_CONTENT, tier.get()); + + settings = Settings.builder().putNull(DataTier.TIER_PREFERENCE).build(); + tier = aggregatedTierPreference(settings, true, true); + assertEquals(DataTier.DATA_HOT, tier.get()); + } + public void testRejectWithSoftDeletesDisabled() { final IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet()); + Collections.emptySet(), randomBoolean()); }); assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " + "Please do not specify a value for setting [index.soft_deletes.enabled].")); @@ -970,7 +1046,7 @@ public void testRejectTranslogRetentionSettings() { IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet())); + Collections.emptySet(), randomBoolean())); assertThat(error.getMessage(), equalTo("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are no longer supported. Please do not specify values for these settings")); } @@ -987,7 +1063,7 @@ public void testDeprecateTranslogRetentionSettings() { request.settings(settings.build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet()); + Collections.emptySet(), randomBoolean()); assertWarnings("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version."); } @@ -1000,7 +1076,7 @@ public void testDeprecateSimpleFS() { request.settings(settings.build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet()); + Collections.emptySet(), randomBoolean()); assertWarnings("[simplefs] is deprecated and will be removed in 8.0. Use [niofs] or other file systems instead. " + "Elasticsearch 7.15 or later uses [niofs] for the [simplefs] store type " + "as it offers superior or equivalent performance to [simplefs]."); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index af6b8d40029da..f9f623979f5af 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.env.Environment; @@ -72,6 +74,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Mockito.mock; public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { @@ -1511,9 +1514,11 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { } private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { + ThreadPool testThreadPool = mock(ThreadPool.class); + ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( Settings.EMPTY, - null, + clusterService, null, null, null, diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index a053ea6f5c2cb..2def60da0a2d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -66,6 +66,7 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; @@ -1753,6 +1754,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS)) // randomly enable low-level search cancellation to make sure it does not alter results .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), randomBoolean()) + // randomly enable enforcing a default tier_preference to make sure it does not alter results + .put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.getKey(), randomBoolean()) .putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()) // empty list disables a port scan for other nodes .putList(DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "file"); return builder.build(); diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java index 5dab6aad05ec5..3e11fc0c6d287 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.autoscaling.storage; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -150,6 +151,11 @@ public void testScaleFromEmptyLegacy() { DiscoveryNodeRole.DATA_HOT_NODE_ROLE ) ); + + // if we inject a default tier preference, then this test wouldn't be valid anymore, + // so let's turn that off + enforceDefaultTierPreference(false); + putAutoscalingPolicy("hot", DataTier.DATA_HOT); putAutoscalingPolicy("warm", DataTier.DATA_WARM); putAutoscalingPolicy("cold", DataTier.DATA_COLD); @@ -243,4 +249,10 @@ private void putAutoscalingPolicy(String policyName, String role) { ); assertAcked(client().execute(PutAutoscalingPolicyAction.INSTANCE, request).actionGet()); } + + public void enforceDefaultTierPreference(boolean enforceDefaultTierPreference) { + ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); + request.transientSettings(Settings.builder().put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE, enforceDefaultTierPreference).build()); + assertAcked(client().admin().cluster().updateSettings(request).actionGet()); + } } diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java index f51817ea63544..df3195b5afbb1 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.cluster.routing.allocation; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; @@ -27,6 +28,7 @@ import java.util.Collections; import java.util.Map; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -88,6 +90,7 @@ public void testRequestSettingOverridesAllocation() { startWarmOnlyNode(); startColdOnlyNode(); ensureGreen(); + enforceDefaultTierPreference(false); client().admin().indices().prepareCreate(index) .setWaitForActiveShards(0) @@ -124,6 +127,25 @@ public void testRequestSettingOverridesAllocation() { ensureYellow(index); } + public void testRequestSettingOverriddenIfEnforced() { + startContentOnlyNode(); + ensureGreen(); + enforceDefaultTierPreference(true); + + client().admin().indices().prepareCreate(index) + .setWaitForActiveShards(0) + .setSettings(Settings.builder() + .putNull(DataTier.TIER_PREFERENCE)) // will be overridden to data_content + .get(); + + Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(DataTier.TIER_PREFERENCE_SETTING.get(idxSettings), equalTo("data_content")); + + // index should be yellow + logger.info("--> waiting for {} to be yellow", index); + ensureYellow(index); + } + /** * When a new index is created from source metadata (as during a shrink), the data tier * default setting should *not* be applied. This test checks that behavior. @@ -164,6 +186,7 @@ public void testShrinkStaysOnTier() { public void testTemplateOverridesDefaults() { startWarmOnlyNode(); + enforceDefaultTierPreference(false); Template t = new Template(Settings.builder() .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".box", "warm") @@ -200,6 +223,28 @@ public void testTemplateOverridesDefaults() { ensureYellow(index); } + public void testTemplateOverriddenIfEnforced() { + startContentOnlyNode(); + enforceDefaultTierPreference(true); + + Template t = new Template(Settings.builder() + .putNull(DataTier.TIER_PREFERENCE) + .build(), null, null); + ComposableIndexTemplate ct = new ComposableIndexTemplate.Builder() + .indexPatterns(Collections.singletonList(index)) + .template(t).build(); + client().execute(PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); + + client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get(); + + Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); + assertThat(DataTier.TIER_PREFERENCE_SETTING.get(idxSettings), equalTo("data_content")); + + // index should be yellow + ensureYellow(index); + } + public void testDataTierTelemetry() { startContentOnlyNode(); startContentOnlyNode(); @@ -331,4 +376,10 @@ public void startFrozenOnlyNode() { .build(); internalCluster().startNode(nodeSettings); } + + public void enforceDefaultTierPreference(boolean enforceDefaultTierPreference) { + ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); + request.transientSettings(Settings.builder().put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE, enforceDefaultTierPreference).build()); + assertAcked(client().admin().cluster().updateSettings(request).actionGet()); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index d87e680260e21..1ff67b73b1d11 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.ssl.SslConfiguration; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.core.Booleans; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; @@ -64,6 +63,7 @@ import org.elasticsearch.snapshots.sourceonly.SourceOnlySnapshotRepository; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; import org.elasticsearch.xpack.cluster.routing.allocation.mapper.DataTierFieldMapper; import org.elasticsearch.xpack.core.action.ReloadAnalyzerAction; diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index ae3f3cb0d3d62..694b06342a75a 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -68,6 +68,10 @@ public void refreshIndexAndStartILM() throws IOException { policy = "policy-" + randomAlphaOfLength(5); alias = "alias-" + randomAlphaOfLength(5); assertOK(client().performRequest(new Request("POST", "_ilm/start"))); + + // we can't have the pre-migration indices getting tier preferences auto-assigned, + // if we did, then we wouldn't really be testing the migration *to* data tiers anymore :D + enforceDefaultTierPreference(false); } @AfterClass @@ -328,4 +332,14 @@ private void createLegacyTemplate(String templateName) throws IOException { templateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); client().performRequest(templateRequest); } + + public void enforceDefaultTierPreference(boolean enforceDefaultTierPreference) throws IOException { + Request request = new Request("PUT", "_cluster/settings"); + request.setJsonEntity("{\n" + + " \"persistent\": {\n" + + " \"" + DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE + "\" : " + enforceDefaultTierPreference + "\n" + + " }\n" + + "}"); + assertOK(client().performRequest(request)); + } }