Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce a mechanism to notify plugin before an index/shard folder is going to be deleted from disk #65926

Merged
merged 13 commits into from
Dec 10, 2020
Prev Previous commit
Next Next commit
deleteLeftoverShardDirectory plumbing
tlrx committed Dec 9, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 81da61f7a20c1165b811fe4c7c5465acac3d574d
5 changes: 3 additions & 2 deletions server/src/main/java/org/elasticsearch/index/IndexModule.java
Original file line number Diff line number Diff line change
@@ -413,7 +413,8 @@ public IndexService newIndexService(IndexService.IndexCreationContext indexCreat
IndicesFieldDataCache indicesFieldDataCache,
NamedWriteableRegistry namedWriteableRegistry,
BooleanSupplier idFieldDataEnabled,
ValuesSourceRegistry valuesSourceRegistry) throws IOException {
ValuesSourceRegistry valuesSourceRegistry,
IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener) throws IOException {
final IndexEventListener eventListener = freeze();
Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory =
indexReaderWrapper.get() == null ? (shard) -> null : indexReaderWrapper.get();
@@ -442,7 +443,7 @@ public IndexService newIndexService(IndexService.IndexCreationContext indexCreat
engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, client, queryCache,
directoryFactory, eventListener, readerWrapperFactory, mapperRegistry, indicesFieldDataCache, searchOperationListeners,
indexOperationListeners, namedWriteableRegistry, idFieldDataEnabled, allowExpensiveQueries, expressionResolver,
valuesSourceRegistry, recoveryStateFactory);
valuesSourceRegistry, recoveryStateFactory, indexFoldersDeletionListener);
success = true;
return indexService;
} finally {
Original file line number Diff line number Diff line change
@@ -115,6 +115,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
private final BitsetFilterCache bitsetFilterCache;
private final NodeEnvironment nodeEnv;
private final ShardStoreDeleter shardStoreDeleter;
private final IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener;
private final IndexStorePlugin.DirectoryFactory directoryFactory;
private final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory;
private final CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper;
@@ -178,7 +179,9 @@ public IndexService(
BooleanSupplier allowExpensiveQueries,
IndexNameExpressionResolver expressionResolver,
ValuesSourceRegistry valuesSourceRegistry,
IndexStorePlugin.RecoveryStateFactory recoveryStateFactory) {
IndexStorePlugin.RecoveryStateFactory recoveryStateFactory,
IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener
) {
super(indexSettings);
this.allowExpensiveQueries = allowExpensiveQueries;
this.indexSettings = indexSettings;
@@ -219,6 +222,7 @@ public IndexService(
}

this.shardStoreDeleter = shardStoreDeleter;
this.indexFoldersDeletionListener = indexFoldersDeletionListener;
this.bigArrays = bigArrays;
this.threadPool = threadPool;
this.scriptService = scriptService;
@@ -414,7 +418,8 @@ public synchronized IndexShard createShard(
} catch (IllegalStateException ex) {
logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
try {
ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings, shardPaths ->
indexFoldersDeletionListener.beforeShardFoldersDeleted(shardId, this.indexSettings, shardPaths));
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
} catch (Exception inner) {
ex.addSuppressed(inner);
13 changes: 10 additions & 3 deletions server/src/main/java/org/elasticsearch/index/shard/ShardPath.java
Original file line number Diff line number Diff line change
@@ -36,6 +36,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;

public final class ShardPath {
public static final String INDEX_FOLDER_NAME = "index";
@@ -172,8 +173,13 @@ public static ShardPath loadShardPath(Logger logger, ShardId shardId, String cus
* This method tries to delete left-over shards where the index name has been reused but the UUID is different
* to allow the new shard to be allocated.
*/
public static void deleteLeftoverShardDirectory(Logger logger, NodeEnvironment env,
ShardLock lock, IndexSettings indexSettings) throws IOException {
public static void deleteLeftoverShardDirectory(
final Logger logger,
final NodeEnvironment env,
final ShardLock lock,
final IndexSettings indexSettings,
final Consumer<Path[]> listener
) throws IOException {
final String indexUUID = indexSettings.getUUID();
final Path[] paths = env.availableShardPaths(lock.getShardId());
for (Path path : paths) {
@@ -183,7 +189,8 @@ public static void deleteLeftoverShardDirectory(Logger logger, NodeEnvironment e
if (load.indexUUID.equals(indexUUID) == false && IndexMetadata.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path);
assert Files.isDirectory(path) : path + " is not a directory";
NodeEnvironment.acquireFSLockForPaths(indexSettings, paths);
NodeEnvironment.acquireFSLockForPaths(indexSettings, path);
listener.accept(new Path[]{path});
IOUtils.rm(path);
}
}
Original file line number Diff line number Diff line change
@@ -679,7 +679,8 @@ private synchronized IndexService createIndexService(IndexService.IndexCreationC
indicesFieldDataCache,
namedWriteableRegistry,
this::isIdFieldDataEnabled,
valuesSourceRegistry
valuesSourceRegistry,
indexFoldersDeletionListeners
);
}

Original file line number Diff line number Diff line change
@@ -95,6 +95,7 @@
import org.hamcrest.Matchers;

import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
@@ -130,6 +131,16 @@ public void addPendingDelete(ShardId shardId, IndexSettings indexSettings) {
}
};

private IndexStorePlugin.IndexFoldersDeletionListener indexDeletionListener = new IndexStorePlugin.IndexFoldersDeletionListener() {
@Override
public void beforeIndexFoldersDeleted(Index index, IndexSettings indexSettings, Path[] indexPaths) {
}

@Override
public void beforeShardFoldersDeleted(ShardId shardId, IndexSettings indexSettings, Path[] shardPaths) {
}
};

private final IndexFieldDataCache.Listener listener = new IndexFieldDataCache.Listener() {};
private MapperRegistry mapperRegistry;
private ThreadPool threadPool;
@@ -169,7 +180,7 @@ public void tearDown() throws Exception {
private IndexService newIndexService(IndexModule module) throws IOException {
return module.newIndexService(CREATE_INDEX, nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays,
threadPool, scriptService, clusterService, null, indicesQueryCache, mapperRegistry,
new IndicesFieldDataCache(settings, listener), writableRegistry(), () -> false, null);
new IndicesFieldDataCache(settings, listener), writableRegistry(), () -> false, null, indexDeletionListener);
}

public void testWrapperIsBound() throws IOException {