Skip to content

Commit

Permalink
Improve shutdown, add more logs.
Browse files Browse the repository at this point in the history 
We get an unexpected 1 minute delay at serverSocket.close. We pack it into a thread now and give it 2 seconds.

Signed-off-by: HenrikJannsen <[email protected]>
  • Loading branch information
@HenrikJannsen
HenrikJannsen committed Aug 11, 2024
1 parent bbd53f1 commit 08cac2e
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 47 deletions.
36 changes: 21 additions & 15 deletions core/src/main/java/bisq/core/app/misc/ExecutableForAppWithP2p.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,23 +203,30 @@ public void gracefulShutDown(ResultHandler resultHandler) {
injector.getInstance(RpcService.class).shutDown();
injector.getInstance(DaoSetup.class).shutDown();
injector.getInstance(ArbitratorManager.class).shutDown();
injector.getInstance(OpenOfferManager.class).shutDown(() -> injector.getInstance(P2PService.class).shutDown(() -> {
injector.getInstance(WalletsSetup.class).shutDownComplete.addListener((ov, o, n) -> {
module.close(injector);

PersistenceManager.flushAllDataToDiskAtShutdown(() -> {
resultHandler.handleResult();
log.info("Graceful shutdown completed. Exiting now.");
UserThread.runAfter(() -> System.exit(BisqExecutable.EXIT_SUCCESS), 1);
injector.getInstance(OpenOfferManager.class).shutDown(() -> {
log.info("OpenOfferManager shutdown done");
injector.getInstance(P2PService.class).shutDown(() -> {
log.info("P2PService shutdown done");
injector.getInstance(WalletsSetup.class).shutDownComplete.addListener((ov, o, n) -> {
log.info("WalletsSetup shutdown done");
module.close(injector);
PersistenceManager.flushAllDataToDiskAtShutdown(() -> {
log.info("flushAllDataToDiskAtShutdown done");
resultHandler.handleResult();
log.info("Graceful shutdown completed. Exiting now.");
UserThread.runAfter(() -> System.exit(BisqExecutable.EXIT_SUCCESS), 1);
});
});

injector.getInstance(WalletsSetup.class).shutDown();
injector.getInstance(BtcWalletService.class).shutDown();
injector.getInstance(BsqWalletService.class).shutDown();
});
injector.getInstance(WalletsSetup.class).shutDown();
injector.getInstance(BtcWalletService.class).shutDown();
injector.getInstance(BsqWalletService.class).shutDown();
}));
});
// we wait max 5 sec.
UserThread.runAfter(() -> {
PersistenceManager.flushAllDataToDiskAtShutdown(() -> {
log.info("flushAllDataToDiskAtShutdown done from timeout");
resultHandler.handleResult();
log.info("Graceful shutdown caused a timeout. Exiting now.");
UserThread.runAfter(() -> System.exit(BisqExecutable.EXIT_SUCCESS), 1);
Expand All @@ -232,11 +239,10 @@ public void gracefulShutDown(ResultHandler resultHandler) {
}, 1);
}
} catch (Throwable t) {
log.debug("App shutdown failed with exception");
t.printStackTrace();
log.warn("App shutdown failed with exception", t);
PersistenceManager.flushAllDataToDiskAtShutdown(() -> {
resultHandler.handleResult();
log.info("Graceful shutdown resulted in an error. Exiting now.");
log.warn("Graceful shutdown resulted in an error. Exiting now.");
UserThread.runAfter(() -> System.exit(BisqExecutable.EXIT_FAILURE), 1);
});

Expand Down
11 changes: 5 additions & 6 deletions p2p/src/main/java/bisq/network/p2p/network/Connection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,8 @@ public void shutDown(CloseConnectionReason closeConnectionReason, @Nullable Runn
stopped = true;
UserThread.execute(() -> doShutDown(closeConnectionReason, shutDownCompleteHandler));
}
}, "Connection:SendCloseConnectionMessage-" + this.uid).start();
}, "Connection:SendCloseConnectionMessage-" + this.uid)
.start();
} else {
stopped = true;
doShutDown(closeConnectionReason, shutDownCompleteHandler);
Expand All @@ -521,22 +522,20 @@ public void shutDown(CloseConnectionReason closeConnectionReason, @Nullable Runn
private void doShutDown(CloseConnectionReason closeConnectionReason, @Nullable Runnable shutDownCompleteHandler) {
// Use UserThread.execute as it's not clear if that is called from a non-UserThread
UserThread.execute(() -> connectionListener.onDisconnect(closeConnectionReason, this));

try {
protoOutputStream.onConnectionShutdown();
socket.close();
} catch (SocketException e) {
log.trace("SocketException at shutdown might be expected {}", e.getMessage());
} catch (IOException e) {
log.error("Exception at shutdown. " + e.getMessage());
e.printStackTrace();
log.error("Exception at shutdown. ", e);
} finally {
capabilitiesListeners.clear();

try {
protoInputStream.close();
} catch (IOException e) {
log.error(e.getMessage());
e.printStackTrace();
} catch (IOException ignore) {
}

Utilities.shutdownAndAwaitTermination(executorService, SHUTDOWN_TIMEOUT, TimeUnit.MILLISECONDS);
Expand Down
1 change: 1 addition & 0 deletions p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,7 @@ public void shutDown(Runnable shutDownCompleteHandler) {
server.shutDown();
server = null;
}
log.info("server shutdown completed");

Set<Connection> allConnections = getAllConnections();
int numConnections = allConnections.size();
Expand Down
66 changes: 40 additions & 26 deletions p2p/src/main/java/bisq/network/p2p/network/Server.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package bisq.network.p2p.network;

import bisq.common.proto.network.NetworkProtoResolver;
import bisq.common.util.SingleThreadExecutorUtils;

import java.net.ServerSocket;
import java.net.Socket;
Expand All @@ -26,7 +27,9 @@
import java.io.IOException;

import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.TimeUnit;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -46,7 +49,7 @@ class Server implements Runnable {
private final Set<Connection> connections = new CopyOnWriteArraySet<>();
private final NetworkProtoResolver networkProtoResolver;
private final Thread serverThread = new Thread(this);

private volatile boolean isStopped;

public Server(ServerSocket serverSocket,
MessageListener messageListener,
Expand All @@ -70,11 +73,11 @@ public void start() {
public void run() {
try {
try {
while (isServerActive()) {
while (!isStopped) {
log.debug("Ready to accept new clients on port " + localPort);
final Socket socket = serverSocket.accept();

if (isServerActive()) {
if (!isStopped) {
log.debug("Accepted new client on localPort/port " + socket.getLocalPort() + "/" + socket.getPort());
InboundConnection connection = new InboundConnection(socket,
messageListener,
Expand All @@ -88,14 +91,14 @@ public void run() {
+ "\nconnection.uid={}", serverSocket.getLocalPort(), socket.getPort(), connection.getUid()
+ "\n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n");

if (isServerActive())
if (!isStopped)
connections.add(connection);
else
connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN);
}
}
} catch (IOException e) {
if (isServerActive())
if (!isStopped)
e.printStackTrace();
}
} catch (Throwable t) {
Expand All @@ -106,27 +109,38 @@ public void run() {

public void shutDown() {
log.info("Server shutdown started");
if (isServerActive()) {
serverThread.interrupt();
connections.forEach(connection -> connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN));

try {
if (!serverSocket.isClosed()) {
serverSocket.close();
}
} catch (SocketException e) {
log.debug("SocketException at shutdown might be expected " + e.getMessage());
} catch (IOException e) {
log.debug("Exception at shutdown. " + e.getMessage());
} finally {
log.debug("Server shutdown complete");
}
} else {
log.warn("stopped already called ast shutdown");
if (isStopped) {
return;
}
isStopped = true;
connections.forEach(connection -> connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN));
if (serverSocket.isClosed()) {
return;
}
try {
CompletableFuture.runAsync(() -> {
try {
// Seems it blocks for 1 minute
log.info("Call serverSocket.close");
serverSocket.close();
log.info("serverSocket.close completed");
} catch (SocketException e) {
log.warn("SocketException at shutdown might be expected " + e.getMessage());
} catch (IOException e) {
log.warn("Exception at shutdown. " + e.getMessage());
} finally {
log.info("Server shutdown complete");
}
}, SingleThreadExecutorUtils.getSingleThreadExecutor("Close serverSocket"))
.orTimeout(2, TimeUnit.SECONDS)
.whenComplete((r, t) -> {
if (t != null) {
log.error("serverSocket.close failed. ", t);
}
})
.join();
} catch (Exception e) {
log.warn("Exception at shutdown. " + e.getMessage());
}
}

private boolean isServerActive() {
return !serverThread.isInterrupted();
}
}
1 change: 1 addition & 0 deletions seednode/src/main/java/bisq/seednode/SeedNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ public void startApplication() {
}

public void shutDown() {
log.info("shutDown");
if (getInventoryRequestHandler != null) {
getInventoryRequestHandler.shutDown();
}
Expand Down

0 comments on commit 08cac2e

Please sign in to comment.