diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfCpuException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfCpuException.groovy index 90f08c25b0..d4fe2268b7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfCpuException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfCpuException.groovy @@ -3,5 +3,5 @@ package nextflow.exception import groovy.transform.InheritConstructors @InheritConstructors -class K8sOutOfCpuException extends RuntimeException { +class K8sOutOfCpuException extends RuntimeException implements TemporaryProblem { } diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfMemoryException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfMemoryException.groovy index 0f6ab9d336..87df1670fe 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfMemoryException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/K8sOutOfMemoryException.groovy @@ -3,5 +3,5 @@ package nextflow.exception import groovy.transform.InheritConstructors @InheritConstructors -class K8sOutOfMemoryException extends RuntimeException { +class K8sOutOfMemoryException extends RuntimeException implements TemporaryProblem { } diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy index d4b3c77200..b98885e5b9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy @@ -28,5 +28,5 @@ import groovy.transform.InheritConstructors */ @InheritConstructors @CompileStatic -class NodeTerminationException extends Exception { +class NodeTerminationException extends Exception implements TemporaryProblem { } diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/TemporaryProblem.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/TemporaryProblem.groovy new file mode 100644 index 0000000000..b7e4f4ef99 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/exception/TemporaryProblem.groovy @@ -0,0 +1,7 @@ +package nextflow.exception + +/** + * Exceptions implementing this interface will always lead to a retry + */ +interface TemporaryProblem { +} diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index f47efbb098..3c50cb27c7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -16,8 +16,6 @@ */ package nextflow.processor -import nextflow.exception.K8sOutOfCpuException -import nextflow.exception.K8sOutOfMemoryException import static nextflow.processor.ErrorStrategy.* @@ -63,7 +61,7 @@ import nextflow.dag.NodeMarker import nextflow.exception.FailedGuardException import nextflow.exception.MissingFileException import nextflow.exception.MissingValueException -import nextflow.exception.NodeTerminationException +import nextflow.exception.TemporaryProblem import nextflow.exception.ProcessException import nextflow.exception.ProcessFailedException import nextflow.exception.ProcessUnrecoverableException @@ -974,16 +972,10 @@ class TaskProcessor { // -- do not recoverable error, just re-throw it if( error instanceof Error ) throw error + boolean temporaryProblem = (error instanceof TemporaryProblem || error?.cause instanceof TemporaryProblem) // -- retry without increasing the error counts - if( task && (error.cause instanceof NodeTerminationException - || error.cause instanceof CloudSpotTerminationException - || error instanceof K8sOutOfCpuException - || error instanceof K8sOutOfMemoryException - ) ) { - if( error.cause instanceof NodeTerminationException - || error instanceof K8sOutOfCpuException - || error instanceof K8sOutOfMemoryException - ) + if( task && ( temporaryProblem || error.cause instanceof CloudSpotTerminationException ) ) { + if( temporaryProblem ) log.info "[$task.hashLog] NOTE: ${error.message} -- Execution is retried" else log.info "[$task.hashLog] NOTE: ${error.message} -- Cause: ${error.cause.message} -- Execution is retried"