diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index fd247a6996cc0..a65314eb5d799 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -1514,9 +1514,14 @@ private NoOpResult innerNoOp(final NoOp noOp) throws IOException { : "Noop tombstone document but _tombstone field is not set [" + doc + " ]"; doc.add(softDeletesField); indexWriter.addDocument(doc); - } catch (Exception ex) { + } catch (final Exception ex) { + /* + * Document level failures when adding a no-op are unexpected, we likely hit something fatal such as the Lucene + * index being corrupt, or the Lucene document limit. We have already issued a sequence number here so this is + * fatal, fail the engine. + */ if (ex instanceof AlreadyClosedException == false && indexWriter.getTragicException() == null) { - throw new AssertionError("noop operation should never fail at document level", ex); + failEngine("no-op origin[" + noOp.origin() + "] seq#[" + noOp.seqNo() + "] failed at document level", ex); } throw ex; } @@ -2862,4 +2867,5 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead // remove live entries in the version map refresh("restore_version_map_and_checkpoint_tracker", SearcherScope.INTERNAL, true); } + } diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 09872b418cc22..9dc1fdf35a91b 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -5997,4 +5997,35 @@ private Map tombstonesInVersionMap(InternalEngine .filter(e -> e.getValue() instanceof DeleteVersionValue) .collect(Collectors.toMap(e -> e.getKey(), e -> (DeleteVersionValue) e.getValue())); } + + public void testNoOpFailure() throws IOException { + engine.close(); + final Settings settings = Settings.builder() + .put(defaultSettings.getSettings()) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build(); + final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( + IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build()); + try (Store store = createStore(); + Engine engine = createEngine( + (dir, iwc) -> new IndexWriter(dir, iwc) { + + @Override + public long addDocument(Iterable doc) throws IOException { + throw new IllegalArgumentException("fatal"); + } + + }, + null, + null, + config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) { + final Engine.NoOp op = new Engine.NoOp(0, 0, PRIMARY, System.currentTimeMillis(), "test"); + final IllegalArgumentException e = expectThrows(IllegalArgumentException. class, () -> engine.noOp(op)); + assertThat(e.getMessage(), equalTo("fatal")); + assertTrue(engine.isClosed.get()); + assertThat(engine.failedEngine.get(), not(nullValue())); + assertThat(engine.failedEngine.get(), instanceOf(IllegalArgumentException.class)); + assertThat(engine.failedEngine.get().getMessage(), equalTo("fatal")); + } + } + }