make retry() more flexible

base/error.jl

@@ -50,45 +50,68 @@ macro assert(ex, msgs...)
     :($(esc(ex)) ? $(nothing) : throw(Main.Base.AssertionError($msg)))
 end
 
-# NOTE: Please keep the constant values specified below in sync with the doc string
 const DEFAULT_RETRY_N = 1
 const DEFAULT_RETRY_ON = e->true
 const DEFAULT_RETRY_MAX_DELAY = 10.0
 
 """
-    retry(f, [retry_on]; n=1, max_delay=10.0) -> Function
+    default_retry_check(state, err::Exception, retry_on::Union{Type,Function}=$(DEFAULT_RETRY_ON);
+            n=$(DEFAULT_RETRY_N), max_delay=$(DEFAULT_RETRY_MAX_DELAY)) -> state, retry_bool
 
-Returns a lambda that retries function `f` up to `n` times in the
-event of an exception. If `retry_on` is a `Type` then retry only
-for exceptions of that type. If `retry_on` is a function
-`test_error(::Exception) -> Bool` then retry only if it is true.
+After initializing with an empty `state`, return `true` up to the first `n`
+times called, and `false` subsequently.  If `retry_on` is a `Type` then return
+`true` only for exceptions of that type; if a `Function`, pass through the
+`Bool` it returns.
 
-The first retry happens after a gap of 50 milliseconds or `max_delay`,
-whichever is lower. Subsequently, the delays between retries are
-exponentially increased with a random factor up to `max_delay`.
+The first return from `default_retry_check` is delayed by 50 milliseconds or
+`max_delay`, whichever is lower.  Subsequent delays are exponentially increased
+with a random factor up to `max_delay`.
 
-**Examples**
+See also [`retry`](@ref).
+
+# Examples
+```
+map(retry(x->x^2, (s,e)->default_retry_check(s,e,n=47)), 1:5)
+```
+"""
+function default_retry_check(state, err::Exception, retry_on::Function=DEFAULT_RETRY_ON;
+            n=DEFAULT_RETRY_N, max_delay=DEFAULT_RETRY_MAX_DELAY)
+    if state == nothing
+        state = [n, min(0.05, max_delay)]   # tried a Dict, but got bootstrap errors during make
+    end
+    (state[1]==0 || (try retry_on(err) end !== true)) && return (state, false)
+    state[1] = state[1] - 1
+    state[2] = min(max_delay, state[2])
+    sleep(state[2] * (0.8 + (rand() * 0.2)))
+    state[2] = state[2] * 5
+    (state, true)
+end
+default_retry_check(state, err, t::Type; kw...) = default_retry_check(state, err, e->isa(e, t); kw...)
+
+"""
+    retry(f::Function, check::Function=default_retry_check) -> Function
+
+Returns an anonymous function that calls function `f`.  Should an exception
+arise, `f` is repeatedly called again each time `check` returns `true`.
+`check` should input a state, which will be `nothing` the first time, and an
+`Exception`.
+
+# Examples
 ```julia
 retry(http_get, e -> e.status == "503")(url)
 retry(read, UVError)(io)
 ```
 """
-function retry(f::Function, retry_on::Function=DEFAULT_RETRY_ON; n=DEFAULT_RETRY_N, max_delay=DEFAULT_RETRY_MAX_DELAY)
+function retry(f::Function, check::Function=default_retry_check)
     (args...) -> begin
-        delay = min(0.05, max_delay)
-        for i = 1:n+1
+        local state = nothing
+        while true
             try
                 return f(args...)
             catch e
-                if i > n || try retry_on(e) end !== true
-                    rethrow(e)
-                end
+                state, retry_or_not = check(state,e)
+                retry_or_not || rethrow(e)
             end
-            delay = min(max_delay, delay)
-            sleep(delay * (0.8 + (rand() * 0.2)))
-            delay = delay * 5
         end
     end
 end
-
-retry(f::Function, t::Type; kw...) = retry(f, e->isa(e, t); kw...)

base/exports.jl

@@ -993,6 +993,7 @@ export
     error,
     rethrow,
     retry,
+    default_retry_check,
     systemerror,
 
 # stack traces

base/pmap.jl

@@ -158,7 +158,8 @@ function wrap_on_error(f, on_error; capture_data=false)
     end
 end
 
-wrap_retry(f, retry_on, n, max_delay) = retry(f, retry_on; n=n, max_delay=max_delay)
+wrap_retry(f, retry_on, n, max_delay) =
+        retry(f, (s,e)->default_retry_check(s,e,retry_on; n=n, max_delay=max_delay))
 
 function wrap_batch(f, p, on_error)
     f = asyncmap_batch(f)

base/workerpool.jl

@@ -208,7 +208,7 @@ end
 """
     remote([::AbstractWorkerPool], f) -> Function
 
-Returns a lambda that executes function `f` on an available worker
+Returns an anonymous function that executes function `f` on an available worker
 using [`remotecall_fetch`](@ref).
 """
 remote(f) = (args...; kwargs...)->remotecall_fetch(f, default_worker_pool(), args...; kwargs...)

test/error.jl

@@ -16,42 +16,42 @@ let
 
     # Success on second attempt
     c = [0]
-    @test retry(foo_error;n=1)(c,1) == 7
+    @test retry(foo_error)(c,1) == 7
     @test c[1] == 2
 
     # 2 failed retry attempts, so exception is raised
     c = [0]
-    ex = try retry(foo_error;n=2)(c,3) catch e; e end
+    ex = try retry(foo_error, (s,e)->default_retry_check(s,e,n=2))(c,3) catch e; e end
     @test ex.msg == "foo"
     @test c[1] == 3
 
     c = [0]
-    ex = try retry(foo_error, ErrorException)(c,2) catch e; e end
+    ex = try retry(foo_error, (s,e)->default_retry_check(s,e,ErrorException))(c,2) catch e; e end
     @test typeof(ex) == ErrorException
     @test ex.msg == "foo"
     @test c[1] == 2
 
     c = [0]
-    ex = try retry(foo_error, e->e.msg == "foo")(c,2) catch e; e end
+    ex = try retry(foo_error, (s,e)->default_retry_check(s,e,e->e.msg == "foo"))(c,2) catch e; e end
     @test typeof(ex) == ErrorException
     @test ex.msg == "foo"
     @test c[1] == 2
 
     # No retry if condition does not match
     c = [0]
-    ex = try retry(foo_error, e->e.msg == "bar"; n=3)(c,2) catch e; e end
+    ex = try retry(foo_error, (s,e)->default_retry_check(s,e,e->e.msg == "bar"))(c,2) catch e; e end
     @test typeof(ex) == ErrorException
     @test ex.msg == "foo"
     @test c[1] == 1
 
     c = [0]
-    ex = try retry(foo_error, e->e.http_status_code == "503")(c,2) catch e; e end
+    ex = try retry(foo_error, (s,e)->default_retry_check(s,e,e->e.http_status_code == "503"))(c,2) catch e; e end
     @test typeof(ex) == ErrorException
     @test ex.msg == "foo"
     @test c[1] == 1
 
     c = [0]
-    ex = try retry(foo_error, SystemError)(c,2) catch e; e end
+    ex = try retry(foo_error, (s,e)->default_retry_check(s,e,SystemError))(c,2) catch e; e end
     @test typeof(ex) == ErrorException
     @test ex.msg == "foo"
     @test c[1] == 1