From 12a1f44eb740d40d6cec6a9ee46d690b9e765745 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 4 Feb 2020 10:35:31 +0100 Subject: [PATCH] [Transform] improve irrecoverable error detection treat resource not found and illegal argument exceptions as irrecoverable error relates #50135 --- .../transforms/ClientTransformIndexer.java | 4 +- .../transforms/TransformIndexer.java | 42 ++++++++++--------- .../utils/ExceptionRootCauseFinder.java | 10 +++-- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index 60cfdaf658024..6e247689ad5f9 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -174,7 +174,7 @@ protected void doNextBulk(BulkRequest request, ActionListener next // Determine whether the failure is irrecoverable (transform should go into failed state) or not (transform increments // the indexing failure counter // and possibly retries) - Exception irrecoverableException = ExceptionRootCauseFinder.getFirstIrrecoverableExceptionFromBulkResponses( + Throwable irrecoverableException = ExceptionRootCauseFinder.getFirstIrrecoverableExceptionFromBulkResponses( deduplicatedFailures.values() ); if (irrecoverableException == null) { @@ -373,7 +373,7 @@ private static String getBulkIndexDetailedFailureMessage(String prefix, Map context.getNumFailureRetries()) { - failIndexer( - "task encountered more than " - + context.getNumFailureRetries() - + " failures; latest failure: " - + ExceptionRootCauseFinder.getDetailedMessage(unwrappedException) - ); - } else { - // Since our schedule fires again very quickly after failures it is possible to run into the same failure numerous - // times in a row, very quickly. We do not want to spam the audit log with repeated failures, so only record the first one - if (e.getMessage().equals(lastAuditedExceptionMessage) == false) { - String message = ExceptionRootCauseFinder.getDetailedMessage(unwrappedException); - - auditor.warning( - getJobId(), - "Transform encountered an exception: " + message + " Will attempt again at next scheduled trigger." + || unwrappedException instanceof TransformConfigReloadingException + || unwrappedException instanceof ResourceNotFoundException + || unwrappedException instanceof IllegalArgumentException) { + failIndexer("task encountered irrecoverable failure: " + e.getMessage()); + } else if (context.getAndIncrementFailureCount() > context.getNumFailureRetries()) { + failIndexer( + "task encountered more than " + + context.getNumFailureRetries() + + " failures; latest failure: " + + ExceptionRootCauseFinder.getDetailedMessage(unwrappedException) ); - lastAuditedExceptionMessage = message; + } else { + // Since our schedule fires again very quickly after failures it is possible to run into the same failure numerous + // times in a row, very quickly. We do not want to spam the audit log with repeated failures, so only record the first one + if (e.getMessage().equals(lastAuditedExceptionMessage) == false) { + String message = ExceptionRootCauseFinder.getDetailedMessage(unwrappedException); + + auditor.warning( + getJobId(), + "Transform encountered an exception: " + message + " Will attempt again at next scheduled trigger." + ); + lastAuditedExceptionMessage = message; + } } - } } /** diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/utils/ExceptionRootCauseFinder.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/utils/ExceptionRootCauseFinder.java index 00e46f0b10e1b..024114a6cb943 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/utils/ExceptionRootCauseFinder.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/utils/ExceptionRootCauseFinder.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.transform.utils; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; @@ -63,10 +64,13 @@ public static String getDetailedMessage(Throwable t) { * @param failures a collection of bulk item responses * @return The first exception considered irrecoverable if there are any, null if no irrecoverable exception found */ - public static Exception getFirstIrrecoverableExceptionFromBulkResponses(Collection failures) { + public static Throwable getFirstIrrecoverableExceptionFromBulkResponses(Collection failures) { for (BulkItemResponse failure : failures) { - if (failure.getFailure().getCause() instanceof MapperParsingException) { - return failure.getFailure().getCause(); + Throwable unwrappedThrowable = org.elasticsearch.ExceptionsHelper.unwrapCause(failure.getFailure().getCause()); + if (unwrappedThrowable instanceof MapperParsingException + || unwrappedThrowable instanceof IllegalArgumentException + || unwrappedThrowable instanceof ResourceNotFoundException) { + return unwrappedThrowable; } }