Searchable snapshot rolling restart

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotEnableAllocationDeciderIntegTests.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.searchablesnapshots;
+
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.mockstore.MockRepository;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class SearchableSnapshotEnableAllocationDeciderIntegTests extends BaseSearchableSnapshotsIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(MockRepository.Plugin.class);
+        return plugins;
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            // Use an unbound cache so we can recover the searchable snapshot completely all the times
+            .put(CacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES))
+            .build();
+    }
+
+    public void testAllocationDisabled() throws Exception {
+        final String restoredIndexName = setupMountedIndex();
+        int numPrimaries = getNumShards(restoredIndexName).numPrimaries;
+        setEnableAllocation(EnableAllocationDecider.Allocation.PRIMARIES);
+        if (randomBoolean()) {
+            setSearchableSnapshotPrimariesAllocation(EnableAllocationDecider.Allocation.NONE);
+        }
+        Set<String> indexNodes = internalCluster().nodesInclude(restoredIndexName);
+        for (String indexNode : indexNodes) {
+            internalCluster().restartNode(indexNode);
+        }
+
+        assertBusy(() -> {
+            ClusterHealthResponse response =
+                client().admin().cluster().health(Requests.clusterHealthRequest(restoredIndexName)).actionGet();
+            assertThat(response.getUnassignedShards(), Matchers.equalTo(numPrimaries));
+        });
+
+        setSearchableSnapshotPrimariesAllocation(EnableAllocationDecider.Allocation.PRIMARIES);
+        ensureGreen(restoredIndexName);
+    }
+
+    public void testAllocationEnabled() throws Exception {
+        final String restoredIndexName = setupMountedIndex();
+        if (randomBoolean()) {
+            setEnableAllocation(EnableAllocationDecider.Allocation.PRIMARIES);
+        }
+        setSearchableSnapshotPrimariesAllocation(EnableAllocationDecider.Allocation.PRIMARIES);
+        Set<String> indexNodes = internalCluster().nodesInclude(restoredIndexName);
+        for (String indexNode : indexNodes) {
+            internalCluster().restartNode(indexNode);
+        }
+
+        ensureGreen(restoredIndexName);
+    }
+
+    private String setupMountedIndex() throws Exception {
+        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        createAndPopulateIndex(indexName, Settings.builder());
+
+        final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        createRepository(repositoryName, "mock");
+
+        final SnapshotId snapshotId = createSnapshot(repositoryName, "snapshot-1", List.of(indexName)).snapshotId();
+        assertAcked(client().admin().indices().prepareDelete(indexName));
+        return mountSnapshot(repositoryName, snapshotId.getName(), indexName, Settings.EMPTY);
+    }
+
+    public void setEnableAllocation(EnableAllocationDecider.Allocation allocation) {
+        setAllocation(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, allocation);
+    }
+
+    public void setSearchableSnapshotPrimariesAllocation(EnableAllocationDecider.Allocation allocation) {
+        setAllocation(SearchableSnapshotEnableAllocationDecider.SEARCHABLE_SNAPSHOTS_ALLOCATION_ENABLE_PRIMARIES_SETTING, allocation);
+    }
+
+    private void setAllocation(Setting<EnableAllocationDecider.Allocation> setting, EnableAllocationDecider.Allocation allocation) {
+        logger.info("--> setting allocation to [{}]", allocation);
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(setting.getKey(), allocation.name())
+                        .build()
+                )
+                .get()
+        );
+    }
+}

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotEnableAllocationDecider.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.searchablesnapshots;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+
+public class SearchableSnapshotEnableAllocationDecider extends AllocationDecider {
+
+    static final String NAME = "searchable_snapshots_enable";
+
+    /**
+     * This setting indicates the behavior of searchable snapshots when cluster.routing.allocation.enable=primaries
+     *
+     */
+    public static final Setting<EnableAllocationDecider.Allocation> SEARCHABLE_SNAPSHOTS_ALLOCATION_ENABLE_PRIMARIES_SETTING =
+        new Setting<>("xpack.searchable.snapshot.allocation.enable.primaries", EnableAllocationDecider.Allocation.NONE.toString(),
+            SearchableSnapshotEnableAllocationDecider::parseSetting,
+            Setting.Property.Dynamic, Setting.Property.NodeScope);
+
+    private static EnableAllocationDecider.Allocation parseSetting(String value) {
+        EnableAllocationDecider.Allocation allocation = EnableAllocationDecider.Allocation.parse(value);
+        if (allocation == EnableAllocationDecider.Allocation.ALL || allocation == EnableAllocationDecider.Allocation.NEW_PRIMARIES) {
+            throw new IllegalArgumentException("[" + SEARCHABLE_SNAPSHOTS_ALLOCATION_ENABLE_PRIMARIES_SETTING.getKey() + "=" + allocation
+                 + "] is not valid");
+        }
+        return allocation;
+    }
+
+    private volatile EnableAllocationDecider.Allocation enableAllocation;
+    private volatile EnableAllocationDecider.Allocation primariesAllocation;
+
+    public SearchableSnapshotEnableAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
+        this.enableAllocation = EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.get(settings);
+        this.primariesAllocation = SEARCHABLE_SNAPSHOTS_ALLOCATION_ENABLE_PRIMARIES_SETTING.get(settings);
+        assertSettingValue();
+        clusterSettings.addSettingsUpdateConsumer(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING,
+            this::setEnableAllocation);
+        clusterSettings.addSettingsUpdateConsumer(SEARCHABLE_SNAPSHOTS_ALLOCATION_ENABLE_PRIMARIES_SETTING,
+            this::setPrimariesAllocation);
+    }
+
+    public void assertSettingValue() {
+        assert this.primariesAllocation != EnableAllocationDecider.Allocation.ALL && this.primariesAllocation != EnableAllocationDecider.Allocation.NEW_PRIMARIES;
+    }
+
+    private void setPrimariesAllocation(EnableAllocationDecider.Allocation allocation) {
+        this.primariesAllocation = allocation;
+        assertSettingValue();
+    }
+
+    private void setEnableAllocation(EnableAllocationDecider.Allocation allocation) {
+        this.enableAllocation = allocation;
+    }
+
+    @Override
+    public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+        return canAllocate(shardRouting, allocation);
+    }
+
+    @Override
+    public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
+        final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
+        if (SearchableSnapshotsConstants.isSearchableSnapshotStore(indexMetadata.getSettings())) {
+            EnableAllocationDecider.Allocation enableAllocation = this.enableAllocation;
+            EnableAllocationDecider.Allocation primariesAllocation = this.primariesAllocation;
+            if (enableAllocation == EnableAllocationDecider.Allocation.PRIMARIES) {
+                if (primariesAllocation == EnableAllocationDecider.Allocation.NONE) {
+                    return allocation.decision(Decision.NO, NAME,
+                        "no allocations of searchable snapshots allowed due to [%s=%s] and [%s=%s]",
+                        EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
+                        enableAllocation,
+                        SEARCHABLE_SNAPSHOTS_ALLOCATION_ENABLE_PRIMARIES_SETTING.getKey(),
+                        primariesAllocation
+                    );
+                } else {
+                    return allocation.decision(Decision.YES, NAME, "decider relies on generic enable decider");
+                }
+            } else {
+                return allocation.decision(Decision.YES, NAME, "decider only active when [%s=primaries]",
+                    EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey());
+            }
+        } else {
+            return allocation.decision(Decision.YES, NAME, "decider only applicable for indices backed by searchable snapshots");
+        }
+    }
+}

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java

@@ -190,7 +190,8 @@ public List<Setting<?>> getSettings() {
             CacheService.SNAPSHOT_CACHE_RANGE_SIZE_SETTING,
             CacheService.SNAPSHOT_CACHE_SYNC_INTERVAL_SETTING,
             CacheService.SNAPSHOT_CACHE_MAX_FILES_TO_SYNC_AT_ONCE_SETTING,
-            CacheService.SNAPSHOT_CACHE_SYNC_SHUTDOWN_TIMEOUT
+            CacheService.SNAPSHOT_CACHE_SYNC_SHUTDOWN_TIMEOUT,
+            SearchableSnapshotEnableAllocationDecider.SEARCHABLE_SNAPSHOTS_ALLOCATION_ENABLE_PRIMARIES_SETTING
         );
     }
 
@@ -334,7 +335,8 @@ protected XPackLicenseState getLicenseState() {
     @Override
     public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
         return List.of(
-            new SearchableSnapshotAllocationDecider(() -> getLicenseState().isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS))
+            new SearchableSnapshotAllocationDecider(() -> getLicenseState().isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS)),
+            new SearchableSnapshotEnableAllocationDecider(settings, clusterSettings)
         );
     }