Skip to content

Commit

Permalink
Introduce searchable snapshots index setting for cascade deletion of …
Browse files Browse the repository at this point in the history 
…snapshots (elastic#74977)

Today there is no relationship between the lifecycle of a 
snapshot mounted as an index and the lifecycle of index 
itself. This lack of relationship makes it possible to delete 
a snapshot in a repository while the mounted index still 
exists, causing the shards to fail in the future.

On the other hand creating a snapshot that contains a 
single index to later mount it as a searchable snapshot 
index becomes more and more natural for users and 
ILM; but it comes with the risk to forget to delete the
 snapshot when the searchable snapshot index is 
deleted from the cluster, "leaking" snapshots in 
repositories.

We'd like to improve the situation and provide a way 
to automatically delete the snapshot when the mounted 
index is deleted. To opt in for this behaviour, a user has 
to enable a specific index setting when mounting the 
snapshot (the proposed name for the setting is 
index.store.snapshot.delete_searchable_snapshot). 

Elasticsearch then verifies that the snapshot to mount 
contains only 1 snapshotted index and that the snapshot 
is not used by another mounted index with a different 
value for the opt in setting, and mounts the snapshot 
with the new private index setting. 

This is the part implemented in this commit.

In follow-up pull requests this index setting will be used 
when the last mounted index is deleted and removed 
from the cluster state in order to add the searchable 
snapshot id to a list of snapshots to delete in the 
repository metadata. Snapshots that are marked as 
"to delete" will not be able to be restored or cloned, and 
Elasticsearch will take care of deleting the snapshot as 
soon as possible. Then ILM will be changed to use this 
setting when mounting a snapshot as a cold or frozen 
index and delete_searchable_snapshot option in ILM 
will be removed. Finally, deleting a snapshot that is 
still used by mounted indices will be prevented.
  • Loading branch information
@tlrx @masseyke
tlrx authored and masseyke committed Jul 16, 2021
1 parent e2c9cc4 commit e585eb5
Show file tree
Hide file tree
Showing 4 changed files with 454 additions and 8 deletions.
131 changes: 125 additions & 6 deletions server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.logging.DeprecationCategory;
Expand Down Expand Up @@ -82,6 +83,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
Expand All @@ -100,6 +102,11 @@
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY;
import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices;
import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE;

Expand Down Expand Up @@ -1015,20 +1022,39 @@ private static IndexMetadata updateIndexSettings(
Settings changeSettings,
String[] ignoreSettings
) {
final Settings settings = indexMetadata.getSettings();
Settings normalizedChangeSettings = Settings.builder()
.put(changeSettings)
.normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX)
.build();
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetadata.getSettings())
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings)
&& IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(changeSettings)
&& IndexSettings.INDEX_SOFT_DELETES_SETTING.get(changeSettings) == false) {
throw new SnapshotRestoreException(
snapshot,
"cannot disable setting [" + IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey() + "] on restore"
);
}
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(settings))) {
final Boolean changed = changeSettings.getAsBoolean(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, null);
if (changed != null) {
final Boolean previous = settings.getAsBoolean(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, null);
if (Objects.equals(previous, changed) == false) {
throw new SnapshotRestoreException(
snapshot,
String.format(
Locale.ROOT,
"cannot change value of [%s] when restoring searchable snapshot [%s:%s] as index %s",
SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION,
snapshot.getRepository(),
snapshot.getSnapshotId().getName(),
indexMetadata.getIndex()
)
);
}
}
}
IndexMetadata.Builder builder = IndexMetadata.builder(indexMetadata);
Settings settings = indexMetadata.getSettings();
Set<String> keyFilters = new HashSet<>();
List<String> simpleMatchPatterns = new ArrayList<>();
for (String ignoredSetting : ignoreSettings) {
Expand Down Expand Up @@ -1147,6 +1173,9 @@ public ClusterState execute(ClusterState currentState) {
resolveSystemIndicesToDelete(currentState, featureStatesToRestore)
);

// List of searchable snapshots indices to restore
final Set<Index> searchableSnapshotsIndices = new HashSet<>();

// Updating cluster state
final Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
Expand Down Expand Up @@ -1236,6 +1265,10 @@ && isSystemIndex(snapshotIndexMetadata) == false) {
: new ShardRestoreStatus(localNodeId)
);
}

if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(updatedIndexMetadata.getSettings()))) {
searchableSnapshotsIndices.add(updatedIndexMetadata.getIndex());
}
}

final ClusterState.Builder builder = ClusterState.builder(currentState);
Expand Down Expand Up @@ -1273,10 +1306,11 @@ && isSystemIndex(snapshotIndexMetadata) == false) {
}

updater.accept(currentState, mdBuilder);
return allocationService.reroute(
builder.metadata(mdBuilder).blocks(blocks).routingTable(rtBuilder.build()).build(),
"restored snapshot [" + snapshot + "]"
);
final ClusterState updatedClusterState = builder.metadata(mdBuilder).blocks(blocks).routingTable(rtBuilder.build()).build();
if (searchableSnapshotsIndices.isEmpty() == false) {
ensureSearchableSnapshotsRestorable(updatedClusterState, snapshotInfo, searchableSnapshotsIndices);
}
return allocationService.reroute(updatedClusterState, "restored snapshot [" + snapshot + "]");
}

private void applyDataStreamRestores(ClusterState currentState, Metadata.Builder mdBuilder) {
Expand Down Expand Up @@ -1494,4 +1528,89 @@ private void ensureValidIndexName(ClusterState currentState, IndexMetadata snaps
createIndexService.validateDotIndex(renamedIndexName, isHidden);
createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetadata.getSettings(), false);
}

private static void ensureSearchableSnapshotsRestorable(
final ClusterState currentState,
final SnapshotInfo snapshotInfo,
final Set<Index> indices
) {
final Metadata metadata = currentState.metadata();
for (Index index : indices) {
final Settings indexSettings = metadata.getIndexSafe(index).getSettings();
assert "snapshot".equals(INDEX_STORE_TYPE_SETTING.get(indexSettings)) : "not a snapshot backed index: " + index;

final String repositoryUuid = indexSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY);
final String repositoryName = indexSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY);
final String snapshotUuid = indexSettings.get(SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY);

final boolean deleteSnapshot = indexSettings.getAsBoolean(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, false);
if (deleteSnapshot && snapshotInfo.indices().size() != 1 && Objects.equals(snapshotUuid, snapshotInfo.snapshotId().getUUID())) {
throw new SnapshotRestoreException(
repositoryName,
snapshotInfo.snapshotId().getName(),
String.format(
Locale.ROOT,
"cannot mount snapshot [%s/%s:%s] as index [%s] with the deletion of snapshot on index removal enabled "
+ "[index.store.snapshot.delete_searchable_snapshot: true]; snapshot contains [%d] indices instead of 1.",
repositoryName,
repositoryUuid,
snapshotInfo.snapshotId().getName(),
index.getName(),
snapshotInfo.indices().size()
)
);
}

for (IndexMetadata other : metadata) {
if (other.getIndex().equals(index)) {
continue; // do not check the searchable snapshot index against itself
}
final Settings otherSettings = other.getSettings();
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(otherSettings)) == false) {
continue; // other index is not a searchable snapshot index, skip
}
final String otherSnapshotUuid = otherSettings.get(SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY);
if (Objects.equals(snapshotUuid, otherSnapshotUuid) == false) {
continue; // other index is backed by a different snapshot, skip
}
final String otherRepositoryUuid = otherSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY);
final String otherRepositoryName = otherSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY);
if (matchRepository(repositoryUuid, repositoryName, otherRepositoryUuid, otherRepositoryName) == false) {
continue; // other index is backed by a snapshot from a different repository, skip
}
final boolean otherDeleteSnap = otherSettings.getAsBoolean(SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION, false);
if (deleteSnapshot != otherDeleteSnap) {
throw new SnapshotRestoreException(
repositoryName,
snapshotInfo.snapshotId().getName(),
String.format(
Locale.ROOT,
"cannot mount snapshot [%s/%s:%s] as index [%s] with [index.store.snapshot.delete_searchable_snapshot: %b]; "
+ "another index %s is mounted with [index.store.snapshot.delete_searchable_snapshot: %b].",
repositoryName,
repositoryUuid,
snapshotInfo.snapshotId().getName(),
index.getName(),
deleteSnapshot,
other.getIndex(),
otherDeleteSnap
)
);
}
}
}
}

private static boolean matchRepository(
String repositoryUuid,
String repositoryName,
String otherRepositoryUuid,
String otherRepositoryName
) {
if (Strings.hasLength(repositoryUuid) && Strings.hasLength(otherRepositoryUuid)) {
return Objects.equals(repositoryUuid, otherRepositoryUuid);
} else {
return Objects.equals(repositoryName, otherRepositoryName);
}
}
}
3 changes: 3 additions & 0 deletions server/src/main/java/org/elasticsearch/snapshots/SearchableSnapshotsSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ public final class SearchableSnapshotsSettings {
public static final String SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY = "index.store.snapshot.partial";
public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY = "index.store.snapshot.repository_name";
public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY = "index.store.snapshot.repository_uuid";
public static final String SEARCHABLE_SNAPSHOTS_SNAPSHOT_NAME_SETTING_KEY = "index.store.snapshot.snapshot_name";
public static final String SEARCHABLE_SNAPSHOTS_SNAPSHOT_UUID_SETTING_KEY = "index.store.snapshot.snapshot_uuid";
public static final String SEARCHABLE_SNAPSHOTS_DELETE_SNAPSHOT_ON_INDEX_DELETION = "index.store.snapshot.delete_searchable_snapshot";

private SearchableSnapshotsSettings() {}

Expand Down
Loading

0 comments on commit e585eb5

Please sign in to comment.