Retry commands in testament again (#16262)

nim-lang · Dec 5, 2020 · d7f2441 · d7f2441
1 parent 8178388
commit d7f2441
Showing 1 changed file with 16 additions and 5 deletions.
diff --git a/testament/categories.nim b/testament/categories.nim
@@ -482,6 +482,17 @@ proc makeSupTest(test, options: string, cat: Category): TTest =
   result.options = options
   result.startTime = epochTime()
 
+const maxRetries = 3
+template retryCommand(call): untyped =
+  var res: typeof(call)
+  var backoff = 1
+  for i in 0..<maxRetries:
+    res = call
+    if res.exitCode == QuitSuccess or i == maxRetries-1: break
+    sleep(backoff * 1000)
+    backoff *= 2
+  res
+
 proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string, part: PkgPart) =
   if nimbleExe == "":
     echo "[Warning] - Cannot run nimble tests: Nimble binary not found."
@@ -503,28 +514,28 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string, p
       let buildPath = packagesDir / name
 
       if not dirExists(buildPath):
-        let (cloneCmd, cloneOutput, cloneStatus) = execCmdEx2("git", ["clone", url, buildPath])
+        let (cloneCmd, cloneOutput, cloneStatus) = retryCommand execCmdEx2("git", ["clone", url, buildPath])
         if cloneStatus != QuitSuccess:
           r.addResult(test, targetC, "", cloneCmd & "\n" & cloneOutput, reInstallFailed)
           continue
 
         if not useHead:
-          let (fetchCmd, fetchOutput, fetchStatus) = execCmdEx2("git", ["fetch", "--tags"], workingDir = buildPath)
+          let (fetchCmd, fetchOutput, fetchStatus) = retryCommand execCmdEx2("git", ["fetch", "--tags"], workingDir = buildPath)
           if fetchStatus != QuitSuccess:
             r.addResult(test, targetC, "", fetchCmd & "\n" & fetchOutput, reInstallFailed)
             continue
 
-          let (describeCmd, describeOutput, describeStatus) = execCmdEx2("git", ["describe", "--tags", "--abbrev=0"], workingDir = buildPath)
+          let (describeCmd, describeOutput, describeStatus) = retryCommand execCmdEx2("git", ["describe", "--tags", "--abbrev=0"], workingDir = buildPath)
           if describeStatus != QuitSuccess:
             r.addResult(test, targetC, "", describeCmd & "\n" & describeOutput, reInstallFailed)
             continue
 
-          let (checkoutCmd, checkoutOutput, checkoutStatus) = execCmdEx2("git", ["checkout", describeOutput.strip], workingDir = buildPath)
+          let (checkoutCmd, checkoutOutput, checkoutStatus) = retryCommand execCmdEx2("git", ["checkout", describeOutput.strip], workingDir = buildPath)
           if checkoutStatus != QuitSuccess:
             r.addResult(test, targetC, "", checkoutCmd & "\n" & checkoutOutput, reInstallFailed)
             continue
 
-        let (installDepsCmd, installDepsOutput, installDepsStatus) = execCmdEx2("nimble", ["install", "--depsOnly", "-y"], workingDir = buildPath)
+        let (installDepsCmd, installDepsOutput, installDepsStatus) = retryCommand execCmdEx2("nimble", ["install", "--depsOnly", "-y"], workingDir = buildPath)
         if installDepsStatus != QuitSuccess:
           r.addResult(test, targetC, "", "installing dependencies failed:\n$ " & installDepsCmd & "\n" & installDepsOutput, reInstallFailed)
           continue