Add build retrials due to cuda thrust + VS2019 flakyness

ci/build_windows.py

@@ -154,30 +154,40 @@ def windows_build(args):
     logging.info("Using vcvars environment:\n{}".format(args.vcvars))
 
     path = args.output
-    os.makedirs(path, exist_ok=True)
-
-    mxnet_root = get_mxnet_root()
-    logging.info("Found MXNet root: {}".format(mxnet_root))
 
-    with remember_cwd():
-        os.chdir(path)
-        cmd = "\"{}\" && cmake -GNinja {} {}".format(
-            args.vcvars,
-            CMAKE_FLAGS[args.flavour], mxnet_root)
-        logging.info("Generating project with CMake:\n{}".format(cmd))
-        check_call(cmd, shell=True)
-
-        cmd = "\"{}\" && ninja".format(args.vcvars)
-        logging.info("Building:\n{}".format(cmd))
-
-        t0 = int(time.time())
-        check_call(cmd, shell=True)
-
-        logging.info(
-            "Build flavour: {} complete in directory: \"{}\"".format(
-                args.flavour, os.path.abspath(path)))
-        logging.info("Build took {}".format(
-            datetime.timedelta(seconds=int(time.time() - t0))))
+    # cuda thrust + VS 2019 is flaky: try multiple times if fail
+    MAXIMUM_TRY = 5
+    build_try = 0
+
+    while build_try < MAXIMUM_TRY:
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        os.makedirs(path, exist_ok=True)
+
+        mxnet_root = get_mxnet_root()
+        logging.info("Found MXNet root: {}".format(mxnet_root))
+
+        with remember_cwd():
+            os.chdir(path)
+            cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
+                                                         CMAKE_FLAGS[args.flavour],
+                                                         mxnet_root)
+            logging.info("Generating project with CMake:\n{}".format(cmd))
+            check_call(cmd, shell=True)
+
+            cmd = "\"{}\" && ninja".format(args.vcvars)
+            logging.info("Building:\n{}".format(cmd))
+
+            t0 = int(time.time())
+            ret = call(cmd, shell=True)
+
+            if ret != 0:
+                build_try += 1
+                logging.info("{} build(s) have failed".format(build_try))
+            else:
+                logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
+                logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
+                break
     windows_package(args)