Store DataTier Preference directly on IndexMetadata

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
 import org.elasticsearch.cluster.routing.IndexRouting;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
@@ -574,6 +575,23 @@ public ImmutableOpenMap<String, AliasMetadata> getAliases() {
         return this.aliases;
     }
 
+    /**
+     * Lazy loaded cache for tier preference setting. We can't eager load this setting because
+     * {@link IndexMetadataVerifier#convertSharedCacheTierPreference(IndexMetadata)} might not have acted on this index yet and thus the
+     * setting validation for this setting could fail for metadata loaded from a snapshot or disk after an upgrade.
+     */
+    private List<String> tierPreference;
+
+    public List<String> getTierPreference() {
+        List<String> tierPreference = this.tierPreference;
+        if (tierPreference != null) {
+            return tierPreference;
+        }
+        tierPreference = DataTier.parseTierList(DataTier.TIER_PREFERENCE_SETTING.get(settings));
+        this.tierPreference = tierPreference;
+        return tierPreference;
+    }
+
     /**
      * Return the concrete mapping for this index or {@code null} if this index has no mappings at all.
      */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java → server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java

@@ -1,33 +1,38 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-package org.elasticsearch.xpack.core;
+package org.elasticsearch.cluster.routing.allocation;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.shard.IndexSettingProvider;
-import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
+import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
 
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 /**
  * The {@code DataTier} class encapsulates the formalization of the "content",
  * "hot", "warm", and "cold" tiers as node roles. In contains the
  * roles themselves as well as helpers for validation and determining if a node
  * has a tier configured.
- *
- * Related:
- * {@link org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider}
  */
 public class DataTier {
 
@@ -39,6 +44,17 @@ public class DataTier {
 
     public static final Set<String> ALL_DATA_TIERS = Set.of(DATA_CONTENT, DATA_HOT, DATA_WARM, DATA_COLD, DATA_FROZEN);
 
+    public static final String TIER_PREFERENCE = "index.routing.allocation.include._tier_preference";
+
+    public static final Setting<String> TIER_PREFERENCE_SETTING = new Setting<>(
+        new Setting.SimpleKey(TIER_PREFERENCE),
+        DataTierSettingValidator::getDefaultTierPreference,
+        Function.identity(),
+        new DataTierSettingValidator(),
+        Setting.Property.Dynamic,
+        Setting.Property.IndexScope
+    );
+
     static {
         for (String tier : ALL_DATA_TIERS) {
             assert tier.equals(DATA_FROZEN) || tier.contains(DATA_FROZEN) == false
@@ -60,7 +76,6 @@ public static boolean validTierName(String tierName) {
     /**
      * Based on the provided target tier it will return a comma separated list of preferred tiers.
      * ie. if `data_cold` is the target tier, it will return `data_cold,data_warm,data_hot`
-     * This is usually used in conjunction with {@link DataTierAllocationDecider#TIER_PREFERENCE_SETTING}
      */
     public static String getPreferredTiersConfiguration(String targetTier) {
         int indexOfTargetTier = ORDERED_FROZEN_TO_HOT_TIERS.indexOf(targetTier);
@@ -115,6 +130,15 @@ public static boolean isFrozenNode(final Set<DiscoveryNodeRole> roles) {
         return roles.contains(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE) || roles.contains(DiscoveryNodeRole.DATA_ROLE);
     }
 
+    public static List<String> parseTierList(String tiers) {
+        if (Strings.hasText(tiers) == false) {
+            // avoid parsing overhead in the null/empty string case
+            return List.of();
+        } else {
+            return List.of(tiers.split(","));
+        }
+    }
+
     /**
      * This setting provider injects the setting allocating all newly created indices with
      * {@code index.routing.allocation.include._tier_preference: "data_hot"} for a data stream index
@@ -128,9 +152,9 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP
         @Override
         public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings indexSettings) {
             Set<String> settings = indexSettings.keySet();
-            if (settings.contains(DataTierAllocationDecider.TIER_PREFERENCE)) {
+            if (settings.contains(TIER_PREFERENCE)) {
                 // just a marker -- this null value will be removed or overridden by the template/request settings
-                return Settings.builder().putNull(DataTierAllocationDecider.TIER_PREFERENCE).build();
+                return Settings.builder().putNull(TIER_PREFERENCE).build();
             } else if (settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".")) ||
                 settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".")) ||
                 settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "."))) {
@@ -142,11 +166,60 @@ public Settings getAdditionalIndexSettings(String indexName, boolean isDataStrea
                 // tier if the index is part of a data stream, the "content"
                 // tier if it is not.
                 if (isDataStreamIndex) {
-                    return Settings.builder().put(DataTierAllocationDecider.TIER_PREFERENCE, DATA_HOT).build();
+                    return Settings.builder().put(TIER_PREFERENCE, DATA_HOT).build();
+                } else {
+                    return Settings.builder().put(TIER_PREFERENCE, DATA_CONTENT).build();
+                }
+            }
+        }
+    }
+
+    private static final class DataTierSettingValidator implements Setting.Validator<String> {
+
+        private static final Collection<Setting<?>> dependencies = List.of(
+            IndexModule.INDEX_STORE_TYPE_SETTING,
+            SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING
+        );
+
+        public static String getDefaultTierPreference(Settings settings) {
+            if (SearchableSnapshotsSettings.isPartialSearchableSnapshotIndex(settings)) {
+                return DATA_FROZEN;
+            } else {
+                return "";
+            }
+        }
+
+        @Override
+        public void validate(String value) {
+            if (Strings.hasText(value)) {
+                for (String s : parseTierList(value)) {
+                    if (validTierName(s) == false) {
+                        throw new IllegalArgumentException(
+                            "invalid tier names found in [" + value + "] allowed values are " + ALL_DATA_TIERS);
+                    }
+                }
+            }
+        }
+
+        @Override
+        public void validate(String value, Map<Setting<?>, Object> settings, boolean exists) {
+            if (exists && value != null) {
+                if (SearchableSnapshotsSettings.isPartialSearchableSnapshotIndex(settings)) {
+                    if (value.equals(DATA_FROZEN) == false) {
+                        throw new IllegalArgumentException("only the [" + DATA_FROZEN +
+                            "] tier preference may be used for partial searchable snapshots (got: [" + value + "])");
+                    }
                 } else {
-                    return Settings.builder().put(DataTierAllocationDecider.TIER_PREFERENCE, DATA_CONTENT).build();
+                    if (value.contains(DATA_FROZEN)) {
+                        throw new IllegalArgumentException("[" + DATA_FROZEN + "] tier can only be used for partial searchable snapshots");
+                    }
                 }
             }
         }
+
+        @Override
+        public Iterator<Setting<?>> settings() {
+            return dependencies.iterator();
+        }
     }
 }

server/src/main/java/org/elasticsearch/snapshots/SearchableSnapshotsSettings.java

@@ -8,14 +8,25 @@
 
 package org.elasticsearch.snapshots;
 
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.Map;
+
 import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 
 public final class SearchableSnapshotsSettings {
 
     public static final String SEARCHABLE_SNAPSHOT_STORE_TYPE = "snapshot";
     public static final String SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY = "index.store.snapshot.partial";
+
+    public static final Setting<Boolean> SNAPSHOT_PARTIAL_SETTING = Setting.boolSetting(
+        SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY,
+        false,
+        Setting.Property.IndexScope,
+        Setting.Property.PrivateIndex,
+        Setting.Property.NotCopyableOnResize
+    );
     public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY = "index.store.snapshot.repository_name";
     public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY = "index.store.snapshot.repository_uuid";
     public static final String SEARCHABLE_SNAPSHOTS_SNAPSHOT_NAME_SETTING_KEY = "index.store.snapshot.snapshot_name";
@@ -31,4 +42,16 @@ public static boolean isSearchableSnapshotStore(Settings indexSettings) {
     public static boolean isPartialSearchableSnapshotIndex(Settings indexSettings) {
         return isSearchableSnapshotStore(indexSettings) && indexSettings.getAsBoolean(SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY, false);
     }
+
+    /**
+     * Based on a map from setting to value, do the settings represent a partial searchable snapshot index?
+     *
+     * Both index.store.type and index.store.snapshot.partial must be supplied.
+     */
+    public static boolean isPartialSearchableSnapshotIndex(Map<Setting<?>, Object> indexSettings) {
+        assert indexSettings.containsKey(INDEX_STORE_TYPE_SETTING) : "must include store type in map";
+        assert indexSettings.get(SNAPSHOT_PARTIAL_SETTING) != null : "partial setting must be non-null in map (has default value)";
+        return SEARCHABLE_SNAPSHOT_STORE_TYPE.equals(indexSettings.get(INDEX_STORE_TYPE_SETTING))
+            && (boolean) indexSettings.get(SNAPSHOT_PARTIAL_SETTING);
+    }
 }

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -22,7 +23,6 @@
 import org.elasticsearch.xpack.autoscaling.action.PutAutoscalingPolicyAction;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCapacity;
 import org.elasticsearch.xpack.autoscaling.shards.LocalStateAutoscalingAndSearchableSnapshots;
-import org.elasticsearch.xpack.core.DataTier;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
 import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java

@@ -16,15 +16,14 @@
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.NodeRoles;
 import org.elasticsearch.xpack.autoscaling.action.GetAutoscalingCapacityAction;
 import org.elasticsearch.xpack.autoscaling.action.PutAutoscalingPolicyAction;
-import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
-import org.elasticsearch.xpack.core.DataTier;
 import org.hamcrest.Matchers;
 
 import java.util.Arrays;
@@ -118,7 +117,7 @@ private void testScaleFromEmptyWarm(boolean allocatable) throws Exception {
                     .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6)
                     .put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms")
-                    .put(DataTierAllocationDecider.TIER_PREFERENCE, allocatable ? "data_hot" : "data_content")
+                    .put(DataTier.TIER_PREFERENCE, allocatable ? "data_hot" : "data_content")
                     .build()
             ).setWaitForActiveShards(allocatable ? ActiveShardCount.DEFAULT : ActiveShardCount.NONE)
         );
@@ -131,9 +130,7 @@ private void testScaleFromEmptyWarm(boolean allocatable) throws Exception {
             client().admin()
                 .indices()
                 .updateSettings(
-                    new UpdateSettingsRequest(indexName).settings(
-                        Settings.builder().put(DataTierAllocationDecider.TIER_PREFERENCE, "data_warm,data_hot")
-                    )
+                    new UpdateSettingsRequest(indexName).settings(Settings.builder().put(DataTier.TIER_PREFERENCE, "data_warm,data_hot"))
                 )
                 .actionGet()
         );

x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -288,7 +289,7 @@ public boolean canRemainOnlyHighestTierPreference(ShardRouting shard, RoutingAll
             ) != Decision.NO;
             if (result
                 && nodes.isEmpty()
-                && Strings.hasText(DataTierAllocationDecider.TIER_PREFERENCE_SETTING.get(indexMetadata(shard, allocation).getSettings()))) {
+                && Strings.hasText(DataTier.TIER_PREFERENCE_SETTING.get(indexMetadata(shard, allocation).getSettings()))) {
                 // The data tier decider allows a shard to remain on a lower preference tier when no nodes exists on higher preference
                 // tiers.
                 // Here we ensure that if our policy governs the highest preference tier, we assume the shard needs to move to that tier
@@ -391,10 +392,9 @@ private IndexMetadata indexMetadata(ShardRouting shard, RoutingAllocation alloca
             return allocation.metadata().getIndexSafe(shard.index());
         }
 
-        private Optional<String> highestPreferenceTier(String tierPreference, DiscoveryNodes nodes) {
-            String[] preferredTiers = DataTierAllocationDecider.parseTierList(tierPreference);
-            assert preferredTiers.length > 0;
-            return Optional.of(preferredTiers[0]);
+        private Optional<String> highestPreferenceTier(List<String> preferredTiers, DiscoveryNodes nodes) {
+            assert preferredTiers.isEmpty() == false;
+            return Optional.of(preferredTiers.get(0));
         }
 
         public long maxShardSize() {

x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/util/FrozenUtils.java

@@ -7,16 +7,17 @@
 
 package org.elasticsearch.xpack.autoscaling.util;
 
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
-import org.elasticsearch.xpack.core.DataTier;
+
+import java.util.List;
 
 public class FrozenUtils {
     public static boolean isFrozenIndex(Settings indexSettings) {
-        String tierPreference = DataTierAllocationDecider.TIER_PREFERENCE_SETTING.get(indexSettings);
-        String[] preferredTiers = DataTierAllocationDecider.parseTierList(tierPreference);
-        if (preferredTiers.length >= 1 && preferredTiers[0].equals(DataTier.DATA_FROZEN)) {
-            assert preferredTiers.length == 1 : "frozen tier preference must be frozen only";
+        String tierPreference = DataTier.TIER_PREFERENCE_SETTING.get(indexSettings);
+        List<String> preferredTiers = DataTier.parseTierList(tierPreference);
+        if (preferredTiers.isEmpty() == false && preferredTiers.get(0).equals(DataTier.DATA_FROZEN)) {
+            assert preferredTiers.size() == 1 : "frozen tier preference must be frozen only";
             return true;
         } else {
             return false;

x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderServiceTests.java

@@ -11,14 +11,14 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderContext;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult;
 import org.elasticsearch.xpack.autoscaling.util.FrozenUtilsTests;
-import org.elasticsearch.xpack.core.DataTier;
 
 import java.util.Objects;
 

x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -23,7 +24,6 @@
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderContext;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult;
 import org.elasticsearch.xpack.autoscaling.util.FrozenUtilsTests;
-import org.elasticsearch.xpack.core.DataTier;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;