Recover from client connection errors

Attempt to fix com-lihaoyi#2805 We now catch the exception, delete the content of the mill-worker-xyx directory and try again. Seems to be working correcly from manual tests
lolgab · Nov 16, 2023 · a7ca2ee · a7ca2ee
1 parent e93f6bd
commit a7ca2ee
Showing 1 changed file with 55 additions and 44 deletions.
diff --git a/main/client/src/mill/main/client/MillClientMain.java b/main/client/src/mill/main/client/MillClientMain.java
@@ -101,47 +101,58 @@ public static int main0(String[] args) throws Exception {
 
         int index = 0;
         while (index < serverProcessesLimit) {
-            index += 1;
-            final String lockBase = "out/mill-worker-" + versionAndJvmHomeEncoding + "-" + index;
-            new java.io.File(lockBase).mkdirs();
-
-            final File stdout = new java.io.File(lockBase + "/stdout");
-            final File stderr = new java.io.File(lockBase + "/stderr");
-            final int refeshIntervalMillis = 2;
-
-            try (
-                Locks locks = Locks.files(lockBase);
-                final FileToStreamTailer stdoutTailer = new FileToStreamTailer(stdout, System.out, refeshIntervalMillis);
-                final FileToStreamTailer stderrTailer = new FileToStreamTailer(stderr, System.err, refeshIntervalMillis);
-            ) {
-                Locked clientLock = locks.clientLock.tryLock();
-                if (clientLock != null) {
-                    stdoutTailer.start();
-                    stderrTailer.start();
-                    final int exitCode = run(
-                        lockBase,
-                        () -> {
-                            try {
-                                initServer(lockBase, setJnaNoSys);
-                            } catch (Exception e) {
-                                throw new RuntimeException(e);
-                            }
-                        },
-                        locks,
-                        System.in,
-                        System.out,
-                        System.err,
-                        args,
-                        System.getenv()
-                    );
-
-                    // Here, we ensure we process the tails of the output files before interrupting the threads
-                    stdoutTailer.flush();
-                    stderrTailer.flush();
-                    clientLock.release();
-                    return exitCode;
+            int attempts = 0;
+            while (attempts < 3) {
+                final String lockBase = "out/mill-worker-" + versionAndJvmHomeEncoding + "-" + index;
+                java.io.File lockBaseFile = new java.io.File(lockBase);
+                try {
+                    lockBaseFile.mkdirs();
+
+                    final File stdout = new java.io.File(lockBase + "/stdout");
+                    final File stderr = new java.io.File(lockBase + "/stderr");
+                    final int refeshIntervalMillis = 2;
+
+                    try (
+                        Locks locks = Locks.files(lockBase);
+                        final FileToStreamTailer stdoutTailer =new FileToStreamTailer(stdout, System.out, refeshIntervalMillis);
+                        final FileToStreamTailer stderrTailer = new FileToStreamTailer(stderr, System.err, refeshIntervalMillis);
+                    ) {
+                        Locked clientLock = locks.clientLock.tryLock();
+                        if (clientLock != null) {
+                            stdoutTailer.start();
+                            stderrTailer.start();
+                            final int exitCode = run(
+                                    lockBase,
+                                    () -> {
+                                        try {
+                                            initServer(lockBase, setJnaNoSys);
+                                        } catch (Exception e) {
+                                            throw new RuntimeException(e);
+                                        }
+                                    },
+                                    locks,
+                                    System.in,
+                                    System.out,
+                                    System.err,
+                                    args,
+                                    System.getenv());
+
+                            // Here, we ensure we process the tails of the output files before interrupting
+                            // the threads
+                            stdoutTailer.flush();
+                            stderrTailer.flush();
+                            clientLock.release();
+                            return exitCode;
+                        }
+                    }
+                } catch (Exception e) {
+                    for (File file : lockBaseFile.listFiles()) {
+                        file.delete();
+                    }
                 }
+                attempts++;
             }
+            index++;
         }
         throw new MillServerCouldNotBeStarted("Reached max server processes limit: " + serverProcessesLimit);
     }
@@ -176,14 +187,14 @@ public static int run(
         }
         while (locks.processLock.probe()) Thread.sleep(3);
 
+        String socketName = lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io";
+        AFUNIXSocketAddress addr = AFUNIXSocketAddress.of(new File(socketName));
+
+        long retryStart = System.currentTimeMillis();
         Socket ioSocket = null;
         Throwable socketThrowable = null;
-        long retryStart = System.currentTimeMillis();
-
-        while (ioSocket == null && System.currentTimeMillis() - retryStart < 5000) {
+        while (ioSocket == null && System.currentTimeMillis() - retryStart < 1000) {
             try {
-                String socketName = lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io";
-                AFUNIXSocketAddress addr = AFUNIXSocketAddress.of(new File(socketName));
                 ioSocket = AFUNIXSocket.connectTo(addr);
             } catch (Throwable e) {
                 socketThrowable = e;