Merge pull request #4047 from freimair/graceful_shutdown

Added graceful shutdown hook

core/src/main/java/bisq/core/app/BisqExecutable.java

@@ -49,6 +49,10 @@
 
 import lombok.extern.slf4j.Slf4j;
 
+
+
+import sun.misc.Signal;
+
 @Slf4j
 public abstract class BisqExecutable implements GracefulShutDownHandler, BisqSetup.BisqSetupListener {
 
@@ -63,6 +67,7 @@ public abstract class BisqExecutable implements GracefulShutDownHandler, BisqSet
     protected Injector injector;
     protected AppModule module;
     protected Config config;
+    private boolean isShutdown = false;
 
     public BisqExecutable(String fullName, String scriptName, String appName, String version) {
         this.fullName = fullName;
@@ -100,6 +105,16 @@ protected void doExecute() {
         CoreSetup.setup(config);
         addCapabilities();
 
+        Signal.handle(new Signal("INT"), signal -> {
+            gracefulShutDown(() -> {
+            });
+        });
+
+        Signal.handle(new Signal("TERM"), signal -> {
+            gracefulShutDown(() -> {
+            });
+        });
+
         // If application is JavaFX application we need to wait until it is initialized
         launchApplication();
     }
@@ -189,6 +204,10 @@ protected void startAppSetup() {
     // This might need to be overwritten in case the application is not using all modules
     @Override
     public void gracefulShutDown(ResultHandler resultHandler) {
+        if (isShutdown) // prevent double cleanup
+            return;
+
+        isShutdown = true;
         try {
             if (injector != null) {
                 injector.getInstance(ArbitratorManager.class).shutDown();
@@ -214,10 +233,14 @@ public void gracefulShutDown(ResultHandler resultHandler) {
                 UserThread.runAfter(() -> {
                     log.warn("Timeout triggered resultHandler");
                     resultHandler.handleResult();
+                    System.exit(0);
                 }, 20);
             } else {
                 log.warn("injector == null triggered resultHandler");
-                UserThread.runAfter(resultHandler::handleResult, 1);
+                UserThread.runAfter(() -> {
+                    resultHandler.handleResult();
+                    System.exit(0);
+                }, 1);
             }
         } catch (Throwable t) {
             log.error("App shutdown failed with exception");

core/src/main/java/bisq/core/app/misc/ExecutableForAppWithP2p.java

@@ -80,17 +80,24 @@ public void gracefulShutDown(ResultHandler resultHandler) {
                 injector.getInstance(OpenOfferManager.class).shutDown(() -> injector.getInstance(P2PService.class).shutDown(() -> {
                     injector.getInstance(WalletsSetup.class).shutDownComplete.addListener((ov, o, n) -> {
                         module.close(injector);
-                        log.debug("Graceful shutdown completed");
                         resultHandler.handleResult();
+                        log.info("Graceful shutdown completed");
+                        System.exit(0);
                     });
                     injector.getInstance(WalletsSetup.class).shutDown();
                     injector.getInstance(BtcWalletService.class).shutDown();
                     injector.getInstance(BsqWalletService.class).shutDown();
                 }));
                 // we wait max 5 sec.
-                UserThread.runAfter(resultHandler::handleResult, 5);
+                UserThread.runAfter(() -> {
+                    resultHandler.handleResult();
+                    System.exit(0);
+                }, 5);
             } else {
-                UserThread.runAfter(resultHandler::handleResult, 1);
+                UserThread.runAfter(() -> {
+                    resultHandler.handleResult();
+                    System.exit(0);
+                }, 1);
             }
         } catch (Throwable t) {
             log.debug("App shutdown failed with exception");

desktop/src/main/java/bisq/desktop/app/BisqApp.java

@@ -151,11 +151,11 @@ public void stop() {
                     .hideCloseButton()
                     .useAnimation(false)
                     .show();
-            UserThread.runAfter(() -> {
+            new Thread(() -> {
                 gracefulShutDownHandler.gracefulShutDown(() -> {
                     log.debug("App shutdown complete");
                 });
-            }, 200, TimeUnit.MILLISECONDS);
+            }).start();
             shutDownRequested = true;
         }
     }

p2p/src/main/java/bisq/network/p2p/network/Connection.java

@@ -284,7 +284,7 @@ public void sendMessage(NetworkEnvelope networkEnvelope) {
                                         if (!stopped) {
                                             synchronized (lock) {
                                                 BundleOfEnvelopes current = queueOfBundles.poll();
-                                                if (current != null) {
+                                                if (current != null && !stopped) {
                                                     if (current.getEnvelopes().size() == 1) {
                                                         protoOutputStream.writeEnvelope(current.getEnvelopes().get(0));
                                                     } else {
@@ -627,6 +627,10 @@ public boolean reportInvalidRequest(RuleViolation ruleViolation) {
     private void handleException(Throwable e) {
         CloseConnectionReason closeConnectionReason;
 
+        // silent fail if we are shutdown
+        if (stopped)
+            return;
+
         if (e instanceof SocketException) {
             if (socket.isClosed())
                 closeConnectionReason = CloseConnectionReason.SOCKET_CLOSED;

p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java

@@ -36,6 +36,9 @@
 import javafx.beans.property.ReadOnlyObjectProperty;
 import javafx.beans.property.SimpleObjectProperty;
 
+import java.time.Duration;
+import java.time.LocalTime;
+
 import java.net.ConnectException;
 import java.net.ServerSocket;
 import java.net.Socket;
@@ -222,7 +225,7 @@ public void onSuccess(Connection connection) {
                 }
 
                 public void onFailure(@NotNull Throwable throwable) {
-                    log.info("onFailure at sendMessage: peersNodeAddress={}\n\tmessage={}\n\tthrowable={}", peersNodeAddress, networkEnvelope.getClass().getSimpleName(), throwable.toString());
+                    log.debug("onFailure at sendMessage: peersNodeAddress={}\n\tmessage={}\n\tthrowable={}", peersNodeAddress, networkEnvelope.getClass().getSimpleName(), throwable.toString());
                     UserThread.execute(() -> resultFuture.setException(throwable));
                 }
             });
@@ -330,7 +333,24 @@ public void shutDown(Runnable shutDownCompleteHandler) {
                 server = null;
             }
 
-            getAllConnections().stream().forEach(c -> c.shutDown(CloseConnectionReason.APP_SHUT_DOWN));
+            getAllConnections().parallelStream().forEach(c -> c.shutDown(CloseConnectionReason.APP_SHUT_DOWN));
+
+            // wait for connections to actually close, c.shutDown may create threads to actually close connections...
+            LocalTime timeout = LocalTime.now().plus(Duration.ofSeconds(15));
+            while (!getAllConnections().isEmpty()) {
+                // check timeout
+                if (timeout.isBefore(LocalTime.now())) {
+                    log.error("Could not close all connections within timeout (" + getAllConnections().size() + " connections remaining). Moving on.");
+                    break;
+                }
+                try {
+                    // reduce system load
+                    Thread.sleep(10);
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+
             log.debug("NetworkNode shutdown complete");
         }
         if (shutDownCompleteHandler != null) shutDownCompleteHandler.run();

p2p/src/main/java/bisq/network/p2p/network/TorNetworkNode.java

@@ -148,8 +148,10 @@ public Socks5Proxy getSocksProxy() {
     }
 
     public void shutDown(@Nullable Runnable shutDownCompleteHandler) {
-        BooleanProperty torNetworkNodeShutDown = torNetworkNodeShutDown();
+        // this one is executed synchronously
         BooleanProperty networkNodeShutDown = networkNodeShutDown();
+        // this one is committed as a thread to the executor
+        BooleanProperty torNetworkNodeShutDown = torNetworkNodeShutDown();
         BooleanProperty shutDownTimerTriggered = shutDownTimerTriggered();
 
         // Need to store allShutDown to not get garbage collected
@@ -185,6 +187,14 @@ private BooleanProperty torNetworkNodeShutDown() {
                 long ts = System.currentTimeMillis();
                 log.debug("Shutdown torNetworkNode");
                 try {
+                    /**
+                     * make sure we get tor.
+                     * - there have been situations where <code>tor</code> isn't set yet, which would leave tor running
+                     * - downside is that if tor is not already started, we start it here just to shut it down. However,
+                     *   that can only be the case if Bisq gets shutdown even before it reaches step 2/4 at startup.
+                     * The risk seems worth it compared to the risk of not shutting down tor.
+                     */
+                    tor = Tor.getDefault();
                     if (tor != null)
                         tor.shutdown();
                     log.debug("Shutdown torNetworkNode done after " + (System.currentTimeMillis() - ts) + " ms.");