From 8121c8cd7e27248693113d5d7bd62c8275bfc07c Mon Sep 17 00:00:00 2001 From: woodser Date: Sat, 13 Jul 2024 13:19:38 -0400 Subject: [PATCH] switch to next best monerod on various errors --- .../haveno/core/api/XmrConnectionService.java | 75 ++++++++- .../haveno/core/offer/OpenOfferManager.java | 1 + .../tasks/MakerReserveOfferFunds.java | 1 + .../arbitration/ArbitrationManager.java | 1 + .../main/java/haveno/core/trade/Trade.java | 154 +++++++++++------- .../tasks/MaybeSendSignContractRequest.java | 1 + .../tasks/TakerReserveTradeFunds.java | 1 + .../core/xmr/wallet/XmrWalletService.java | 68 +++++--- .../main/funds/withdrawal/WithdrawalView.java | 1 + 9 files changed, 213 insertions(+), 90 deletions(-) diff --git a/core/src/main/java/haveno/core/api/XmrConnectionService.java b/core/src/main/java/haveno/core/api/XmrConnectionService.java index 0c78fc9875f..633e05b1c1e 100644 --- a/core/src/main/java/haveno/core/api/XmrConnectionService.java +++ b/core/src/main/java/haveno/core/api/XmrConnectionService.java @@ -36,7 +36,10 @@ import haveno.network.p2p.P2PService; import haveno.network.p2p.P2PServiceListener; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import javafx.beans.property.IntegerProperty; import javafx.beans.property.LongProperty; @@ -103,6 +106,12 @@ public final class XmrConnectionService { private boolean isShutDownStarted; private List listeners = new ArrayList<>(); + // connection switching + private static final int EXCLUDE_CONNECTION_SECONDS = 300; + private static final int SKIP_SWITCH_WITHIN_MS = 60000; + private Set excludedConnections = new HashSet<>(); + private long lastSwitchRequestTimestamp; + @Inject public XmrConnectionService(P2PService p2PService, Config config, @@ -201,12 +210,6 @@ public List getConnections() { return connectionManager.getConnections(); } - public void switchToBestConnection() { - if (isFixedConnection() || !connectionManager.getAutoSwitch()) return; - MoneroRpcConnection bestConnection = getBestAvailableConnection(); - if (bestConnection != null) setConnection(bestConnection); - } - public void setConnection(String connectionUri) { accountService.checkAccountOpen(); connectionManager.setConnection(connectionUri); // listener will update connection list @@ -244,10 +247,67 @@ public void stopCheckingConnection() { public MoneroRpcConnection getBestAvailableConnection() { accountService.checkAccountOpen(); List ignoredConnections = new ArrayList(); - if (xmrLocalNode.shouldBeIgnored() && connectionManager.hasConnection(xmrLocalNode.getUri())) ignoredConnections.add(connectionManager.getConnectionByUri(xmrLocalNode.getUri())); + addLocalNodeIfIgnored(ignoredConnections); return connectionManager.getBestAvailableConnection(ignoredConnections.toArray(new MoneroRpcConnection[0])); } + private MoneroRpcConnection getBestAvailableConnection(Collection ignoredConnections) { + accountService.checkAccountOpen(); + Set ignoredConnectionsSet = new HashSet<>(ignoredConnections); + addLocalNodeIfIgnored(ignoredConnectionsSet); + return connectionManager.getBestAvailableConnection(ignoredConnectionsSet.toArray(new MoneroRpcConnection[0])); + } + + private void addLocalNodeIfIgnored(Collection ignoredConnections) { + if (xmrLocalNode.shouldBeIgnored() && connectionManager.hasConnection(xmrLocalNode.getUri())) ignoredConnections.add(connectionManager.getConnectionByUri(xmrLocalNode.getUri())); + } + + private void switchToBestConnection() { + if (isFixedConnection() || !connectionManager.getAutoSwitch()) { + log.info("Skipping switch to best Monero connection because connection is fixed or auto switch is disabled"); + return; + } + MoneroRpcConnection bestConnection = getBestAvailableConnection(); + if (bestConnection != null) setConnection(bestConnection); + } + + public boolean requestSwitchToNextBestConnection() { + log.info("Requesting switch to next best Monero connection"); + + // skip if connection is fixed + if (isFixedConnection() || !connectionManager.getAutoSwitch()) { + log.info("Skipping switch to next best Monero connection because connection is fixed or auto switch is disabled"); + return false; + } + + // skip if last switch was too recent + boolean skipSwitch = System.currentTimeMillis() - lastSwitchRequestTimestamp < SKIP_SWITCH_WITHIN_MS; + lastSwitchRequestTimestamp = System.currentTimeMillis(); + if (skipSwitch) { + log.warn("Skipping switch to next best Monero connection because last switch was less than {} seconds ago", SKIP_SWITCH_WITHIN_MS / 1000); + lastSwitchRequestTimestamp = System.currentTimeMillis(); + return false; + } + + // try to get connection to switch to + MoneroRpcConnection currentConnection = getConnection(); + if (currentConnection != null) excludedConnections.add(currentConnection); + MoneroRpcConnection bestConnection = getBestAvailableConnection(excludedConnections); + + // remove from excluded connections after period + UserThread.runAfter(() -> { + if (currentConnection != null) excludedConnections.remove(currentConnection); + }, EXCLUDE_CONNECTION_SECONDS); + + // switch to best connection + if (bestConnection == null) { + log.warn("Could not get connection to switch to"); + return false; + } + setConnection(bestConnection); + return true; + } + public void setAutoSwitch(boolean autoSwitch) { accountService.checkAccountOpen(); connectionManager.setAutoSwitch(autoSwitch); @@ -505,7 +565,6 @@ public void onConnectionChanged(MoneroRpcConnection connection) { // register connection listener connectionManager.addListener(this::onConnectionChanged); - isInitialized = true; } diff --git a/core/src/main/java/haveno/core/offer/OpenOfferManager.java b/core/src/main/java/haveno/core/offer/OpenOfferManager.java index e18d06ae9d7..2f1beef8539 100644 --- a/core/src/main/java/haveno/core/offer/OpenOfferManager.java +++ b/core/src/main/java/haveno/core/offer/OpenOfferManager.java @@ -1057,6 +1057,7 @@ private MoneroTxWallet splitAndSchedule(OpenOffer openOffer) { } catch (Exception e) { log.warn("Error creating split output tx to fund offer {} at subaddress {}, attempt={}/{}, error={}", openOffer.getShortId(), entry.getSubaddressIndex(), i + 1, TradeProtocol.MAX_ATTEMPTS, e.getMessage()); if (stopped || i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (xmrConnectionService.isConnected()) xmrConnectionService.requestSwitchToNextBestConnection(); HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } diff --git a/core/src/main/java/haveno/core/offer/placeoffer/tasks/MakerReserveOfferFunds.java b/core/src/main/java/haveno/core/offer/placeoffer/tasks/MakerReserveOfferFunds.java index 61cafd65829..e4623945c87 100644 --- a/core/src/main/java/haveno/core/offer/placeoffer/tasks/MakerReserveOfferFunds.java +++ b/core/src/main/java/haveno/core/offer/placeoffer/tasks/MakerReserveOfferFunds.java @@ -89,6 +89,7 @@ protected void run() { log.warn("Error creating reserve tx, attempt={}/{}, offerId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, openOffer.getShortId(), e.getMessage()); if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; model.getProtocol().startTimeoutTimer(); // reset protocol timeout + if (model.getXmrWalletService().getConnectionService().isConnected()) model.getXmrWalletService().getConnectionService().requestSwitchToNextBestConnection(); HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } diff --git a/core/src/main/java/haveno/core/support/dispute/arbitration/ArbitrationManager.java b/core/src/main/java/haveno/core/support/dispute/arbitration/ArbitrationManager.java index 42b701d3618..4e3c38ae366 100644 --- a/core/src/main/java/haveno/core/support/dispute/arbitration/ArbitrationManager.java +++ b/core/src/main/java/haveno/core/support/dispute/arbitration/ArbitrationManager.java @@ -478,6 +478,7 @@ private MoneroTxSet signAndPublishDisputePayoutTx(Trade trade) { if (trade.isPayoutPublished()) throw new IllegalStateException("Payout tx already published for " + trade.getClass().getSimpleName() + " " + trade.getShortId()); log.warn("Failed to submit dispute payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, trade.getShortId(), e.getMessage()); if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (trade.getXmrConnectionService().isConnected()) trade.requestSwitchToNextBestConnection(); HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } diff --git a/core/src/main/java/haveno/core/trade/Trade.java b/core/src/main/java/haveno/core/trade/Trade.java index 6aa00373070..b1d84d35b42 100644 --- a/core/src/main/java/haveno/core/trade/Trade.java +++ b/core/src/main/java/haveno/core/trade/Trade.java @@ -125,6 +125,7 @@ import java.util.Date; import java.util.List; import java.util.Optional; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; @@ -627,9 +628,7 @@ public void initialize(ProcessModelServiceProvider serviceProvider) { // handle connection change on dedicated thread xmrConnectionService.addConnectionListener(connection -> { - ThreadUtils.submitToPool(() -> { // TODO: remove this? - ThreadUtils.execute(() -> onConnectionChanged(connection), getConnectionChangedThreadId()); - }); + ThreadUtils.execute(() -> onConnectionChanged(connection), getConnectionChangedThreadId()); }); // reset buyer's payment sent state if no ack receive @@ -843,6 +842,14 @@ public boolean isWalletConnectedToDaemon() { } } + public void requestSwitchToNextBestConnection() { + if (xmrConnectionService.requestSwitchToNextBestConnection()) { + CountDownLatch latch = new CountDownLatch(1); + ThreadUtils.execute(() -> latch.countDown(), getConnectionChangedThreadId()); // wait for connection change to complete + HavenoUtils.awaitLatch(latch); + } + } + public boolean isIdling() { return this instanceof ArbitratorTrade && isDepositsConfirmed() && walletExists() && pollNormalStartTimeMs == null; // arbitrator idles trade after deposits confirm unless overriden } @@ -881,20 +888,21 @@ public void pollWalletNormallyForMs(long pollNormalDuration) { } public void importMultisigHex() { - synchronized (walletLock) { - synchronized (HavenoUtils.getDaemonLock()) { // lock on daemon because import calls full refresh - for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { - try { + for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { + try { + synchronized (walletLock) { + synchronized (HavenoUtils.getDaemonLock()) { // lock on daemon because import calls full refresh doImportMultisigHex(); break; - } catch (IllegalArgumentException e) { - throw e; - } catch (Exception e) { - log.warn("Failed to import multisig hex, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); - if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; - HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + log.warn("Failed to import multisig hex, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); + if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection(); + HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } } @@ -1112,20 +1120,21 @@ public MoneroTxWallet createPayoutTx() { verifyDaemonConnection(); // create payout tx - synchronized (walletLock) { - synchronized (HavenoUtils.getWalletFunctionLock()) { - for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { - try { + for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { + try { + synchronized (walletLock) { + synchronized (HavenoUtils.getWalletFunctionLock()) { return doCreatePayoutTx(); - } catch (Exception e) { - log.warn("Failed to create payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); - if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; - HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } - throw new RuntimeException("Failed to create payout tx for " + getClass().getSimpleName() + " " + getId()); + } catch (Exception e) { + log.warn("Failed to create payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); + if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection(); + HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } + throw new RuntimeException("Failed to create payout tx for " + getClass().getSimpleName() + " " + getId()); } private MoneroTxWallet doCreatePayoutTx() { @@ -1133,11 +1142,12 @@ private MoneroTxWallet doCreatePayoutTx() { // check if multisig import needed if (wallet.isMultisigImportNeeded()) throw new RuntimeException("Cannot create payout tx because multisig import is needed"); - // TODO: wallet sometimes returns empty data, after disconnect? - List txs = wallet.getTxs(); // TODO: this fetches from pool - if (txs.isEmpty()) { - log.warn("Restarting wallet for {} {} because deposit txs are missing to create payout tx", getClass().getSimpleName(), getId()); + // TODO: wallet sometimes returns empty data, due to unreliable connection with specific daemons? + if (getMakerDepositTx() == null || getTakerDepositTx() == null) { + log.warn("Switching monerod and restarting trade wallet because deposit txs are missing to create payout tx for {} {}", getClass().getSimpleName(), getShortId()); + requestSwitchToNextBestConnection(); forceRestartTradeWallet(); + if (getMakerDepositTx() == null || getTakerDepositTx() == null) throw new RuntimeException("Deposit txs are still missing after switching monerod and restarting trade wallet to process payout tx for " + getClass().getSimpleName() + " " + getShortId()); } // gather info @@ -1172,21 +1182,22 @@ private MoneroTxWallet doCreatePayoutTx() { } public MoneroTxWallet createDisputePayoutTx(MoneroTxConfig txConfig) { - synchronized (walletLock) { - synchronized (HavenoUtils.getWalletFunctionLock()) { - for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { - try { + for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { + try { + synchronized (walletLock) { + synchronized (HavenoUtils.getWalletFunctionLock()) { return createTx(txConfig); - } catch (Exception e) { - if (e.getMessage().contains("not possible")) throw new RuntimeException("Loser payout is too small to cover the mining fee"); - log.warn("Failed to create dispute payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); - if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; - HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } - throw new RuntimeException("Failed to create payout tx for " + getClass().getSimpleName() + " " + getId()); + } catch (Exception e) { + if (e.getMessage().contains("not possible")) throw new RuntimeException("Loser payout is too small to cover the mining fee"); + log.warn("Failed to create dispute payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); + if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection(); + HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } + throw new RuntimeException("Failed to create payout tx for " + getClass().getSimpleName() + " " + getId()); } /** @@ -1197,23 +1208,33 @@ public MoneroTxWallet createDisputePayoutTx(MoneroTxConfig txConfig) { * @param publish publishes the signed payout tx if true */ public void processPayoutTx(String payoutTxHex, boolean sign, boolean publish) { - log.info("Processing payout tx for {} {}", getClass().getSimpleName(), getId()); - - // TODO: wallet sometimes returns empty data, after disconnect? detect this condition with failure tolerance for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { try { - List txs = wallet.getTxs(); // TODO: this fetches from pool - if (txs.isEmpty()) { - log.warn("Restarting wallet for {} {} because deposit txs are missing to process payout tx", getClass().getSimpleName(), getId()); - forceRestartTradeWallet(); + synchronized (walletLock) { + synchronized (HavenoUtils.getWalletFunctionLock()) { + doProcessPayoutTx(payoutTxHex, sign, publish); + break; + } } - break; } catch (Exception e) { - log.warn("Failed get wallet txs, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); + log.warn("Failed to process payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (xmrConnectionService.isConnected()) requestSwitchToNextBestConnection(); HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } } + } + + private void doProcessPayoutTx(String payoutTxHex, boolean sign, boolean publish) { + log.info("Processing payout tx for {} {}", getClass().getSimpleName(), getId()); + + // TODO: wallet sometimes returns empty data, due to unreliable connection with specific daemons? + if (getMakerDepositTx() == null || getTakerDepositTx() == null) { + log.warn("Switching monerod and restarting trade wallet because deposit txs are missing to process payout tx for {} {}", getClass().getSimpleName(), getShortId()); + requestSwitchToNextBestConnection(); + forceRestartTradeWallet(); + if (getMakerDepositTx() == null || getTakerDepositTx() == null) throw new RuntimeException("Deposit txs are still missing after switching monerod and restarting trade wallet to process payout tx for " + getClass().getSimpleName() + " " + getShortId()); + } // gather relevant info MoneroWallet wallet = getWallet(); @@ -1226,6 +1247,7 @@ public void processPayoutTx(String payoutTxHex, boolean sign, boolean publish) { MoneroTxSet describedTxSet = wallet.describeTxSet(new MoneroTxSet().setMultisigTxHex(payoutTxHex)); if (describedTxSet.getTxs() == null || describedTxSet.getTxs().size() != 1) throw new IllegalArgumentException("Bad payout tx"); // TODO (woodser): test nack MoneroTxWallet payoutTx = describedTxSet.getTxs().get(0); + if (payoutTxId == null) updatePayout(payoutTx); // update payout tx if not signed // verify payout tx has exactly 2 destinations if (payoutTx.getOutgoingTransfer() == null || payoutTx.getOutgoingTransfer().getDestinations() == null || payoutTx.getOutgoingTransfer().getDestinations().size() != 2) throw new IllegalArgumentException("Payout tx does not have exactly two destinations"); @@ -1257,10 +1279,11 @@ public void processPayoutTx(String payoutTxHex, boolean sign, boolean publish) { if (!sellerPayoutDestination.getAmount().equals(expectedSellerPayout)) throw new IllegalArgumentException("Seller destination amount is not deposit amount - trade amount - 1/2 tx costs, " + sellerPayoutDestination.getAmount() + " vs " + expectedSellerPayout); // check connection - if (sign || publish) verifyDaemonConnection(); + boolean doSign = sign && getPayoutTxHex() == null; + if (doSign || publish) verifyDaemonConnection(); // handle tx signing - if (sign) { + if (doSign) { // sign tx try { @@ -1275,6 +1298,7 @@ public void processPayoutTx(String payoutTxHex, boolean sign, boolean publish) { // describe result describedTxSet = wallet.describeMultisigTxSet(payoutTxHex); payoutTx = describedTxSet.getTxs().get(0); + updatePayout(payoutTx); // verify fee is within tolerance by recreating payout tx // TODO (monero-project): creating tx will require exchanging updated multisig hex if message needs reprocessed. provide weight with describe_transfer so fee can be estimated? @@ -1286,22 +1310,16 @@ public void processPayoutTx(String payoutTxHex, boolean sign, boolean publish) { log.info("Payout tx fee {} is within tolerance, diff %={}", payoutTx.getFee(), feeDiff); } - // update trade state - updatePayout(payoutTx); + // save trade state requestPersistence(); // submit payout tx if (publish) { - for (int i = 0; i < TradeProtocol.MAX_ATTEMPTS; i++) { - try { - wallet.submitMultisigTxHex(payoutTxHex); - ThreadUtils.submitToPool(() -> pollWallet()); - break; - } catch (Exception e) { - log.warn("Failed to submit payout tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, getShortId(), e.getMessage()); - if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; - HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying - } + try { + wallet.submitMultisigTxHex(payoutTxHex); + ThreadUtils.submitToPool(() -> pollWallet()); + } catch (Exception e) { + throw new RuntimeException("Failed to submit payout tx for " + getClass().getSimpleName() + " " + getId(), e); } } } @@ -2236,10 +2254,6 @@ private void doPublishTradeStatistics() { // Private /////////////////////////////////////////////////////////////////////////////////////////// - private String getConnectionChangedThreadId() { - return getId() + ".onConnectionChanged"; - } - // lazy initialization private ObjectProperty getAmountProperty() { if (tradeAmountProperty == null) @@ -2255,6 +2269,10 @@ private ObjectProperty getVolumeProperty() { return tradeVolumeProperty; } + private String getConnectionChangedThreadId() { + return getId() + ".onConnectionChanged"; + } + private void onConnectionChanged(MoneroRpcConnection connection) { synchronized (walletLock) { @@ -2464,7 +2482,17 @@ private void doPollWallet() { try { wallet.rescanSpent(); } catch (Exception e) { + + // switch connection on error log.warn("Error rescanning spent outputs to detect payout tx for {} {}, errorMessage={}", getClass().getSimpleName(), getShortId(), e.getMessage()); + requestSwitchToNextBestConnection(); + + // retry scan + try { + wallet.rescanSpent(); + } catch (Exception e2) { + log.warn("Error rescanning spent outputs to detect payout tx for {} {}, errorMessage={}", getClass().getSimpleName(), getShortId(), e2.getMessage()); + } } } diff --git a/core/src/main/java/haveno/core/trade/protocol/tasks/MaybeSendSignContractRequest.java b/core/src/main/java/haveno/core/trade/protocol/tasks/MaybeSendSignContractRequest.java index 353c167ae84..c2769d040e7 100644 --- a/core/src/main/java/haveno/core/trade/protocol/tasks/MaybeSendSignContractRequest.java +++ b/core/src/main/java/haveno/core/trade/protocol/tasks/MaybeSendSignContractRequest.java @@ -105,6 +105,7 @@ protected void run() { } catch (Exception e) { log.warn("Error creating deposit tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, trade.getShortId(), e.getMessage()); if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (trade.getXmrConnectionService().isConnected()) trade.requestSwitchToNextBestConnection(); HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } diff --git a/core/src/main/java/haveno/core/trade/protocol/tasks/TakerReserveTradeFunds.java b/core/src/main/java/haveno/core/trade/protocol/tasks/TakerReserveTradeFunds.java index 6e0719c1543..1afa38eb2e8 100644 --- a/core/src/main/java/haveno/core/trade/protocol/tasks/TakerReserveTradeFunds.java +++ b/core/src/main/java/haveno/core/trade/protocol/tasks/TakerReserveTradeFunds.java @@ -71,6 +71,7 @@ protected void run() { } catch (Exception e) { log.warn("Error creating reserve tx, attempt={}/{}, tradeId={}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, trade.getShortId(), e.getMessage()); if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (trade.getXmrConnectionService().isConnected()) trade.requestSwitchToNextBestConnection(); HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } diff --git a/core/src/main/java/haveno/core/xmr/wallet/XmrWalletService.java b/core/src/main/java/haveno/core/xmr/wallet/XmrWalletService.java index 17883a36b30..406f53eef0a 100644 --- a/core/src/main/java/haveno/core/xmr/wallet/XmrWalletService.java +++ b/core/src/main/java/haveno/core/xmr/wallet/XmrWalletService.java @@ -24,6 +24,7 @@ import common.utils.JsonUtils; import haveno.common.ThreadUtils; +import haveno.common.Timer; import haveno.common.UserThread; import haveno.common.config.Config; import haveno.common.file.FileUtil; @@ -67,6 +68,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; import javafx.beans.property.LongProperty; @@ -155,14 +157,16 @@ public class XmrWalletService { private TradeManager tradeManager; private MoneroWallet wallet; public static final Object WALLET_LOCK = new Object(); - private boolean wasWalletSynced = false; + private boolean wasWalletSynced; private final Map> txCache = new HashMap>(); - private boolean isClosingWallet = false; - private boolean isShutDownStarted = false; + private boolean isClosingWallet; + private boolean isShutDownStarted; private ExecutorService syncWalletThreadPool = Executors.newFixedThreadPool(10); // TODO: adjust based on connection type - private Long syncStartHeight = null; - private TaskLooper syncWithProgressLooper = null; - CountDownLatch syncWithProgressLatch; + private Long syncStartHeight; + private TaskLooper syncProgressLooper; + private CountDownLatch syncProgressLatch; + private Timer syncProgressTimeout; + private static final int SYNC_PROGRESS_TIMEOUT_SECONDS = 45; // wallet polling and cache private TaskLooper pollLooper; @@ -1431,6 +1435,15 @@ private void resetIfWalletChanged() { private void syncWithProgress() { + // start sync progress timeout + resetSyncProgressTimeout(); + + // switch connection if disconnected + if (!wallet.isConnectedToDaemon()) { + log.warn("Switching connection before syncing with progress because disconnected"); + if (xmrConnectionService.requestSwitchToNextBestConnection()) return; + } + // show sync progress updateSyncProgress(wallet.getHeight()); @@ -1458,8 +1471,8 @@ public void onSyncProgress(long height, long startHeight, long endHeight, double // poll wallet for progress wallet.startSyncing(xmrConnectionService.getRefreshPeriodMs()); - syncWithProgressLatch = new CountDownLatch(1); - syncWithProgressLooper = new TaskLooper(() -> { + syncProgressLatch = new CountDownLatch(1); + syncProgressLooper = new TaskLooper(() -> { if (wallet == null) return; long height = 0; try { @@ -1470,29 +1483,30 @@ public void onSyncProgress(long height, long startHeight, long endHeight, double } if (height < xmrConnectionService.getTargetHeight()) updateSyncProgress(height); else { - syncWithProgressLooper.stop(); + syncProgressLooper.stop(); wasWalletSynced = true; updateSyncProgress(height); - syncWithProgressLatch.countDown(); + syncProgressLatch.countDown(); } }); - syncWithProgressLooper.start(1000); - HavenoUtils.awaitLatch(syncWithProgressLatch); + syncProgressLooper.start(1000); + HavenoUtils.awaitLatch(syncProgressLatch); wallet.stopSyncing(); if (!wasWalletSynced) throw new IllegalStateException("Failed to sync wallet with progress"); } private void stopSyncWithProgress() { - if (syncWithProgressLooper != null) { - syncWithProgressLooper.stop(); - syncWithProgressLooper = null; - syncWithProgressLatch.countDown(); + if (syncProgressLooper != null) { + syncProgressLooper.stop(); + syncProgressLooper = null; + syncProgressLatch.countDown(); } } private void updateSyncProgress(long height) { UserThread.execute(() -> { walletHeight.set(height); + resetSyncProgressTimeout(); // new wallet reports height 1 before synced if (height == 1) { @@ -1509,6 +1523,16 @@ private void updateSyncProgress(long height) { }); } + private synchronized void resetSyncProgressTimeout() { + if (syncProgressTimeout != null) syncProgressTimeout.stop(); + syncProgressTimeout = UserThread.runAfter(() -> { + if (wasWalletSynced) return; + log.warn("Sync progress timeout called"); + xmrConnectionService.requestSwitchToNextBestConnection(); + resetSyncProgressTimeout(); + }, SYNC_PROGRESS_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + private MoneroWalletFull createWalletFull(MoneroWalletConfig config) { // must be connected to daemon @@ -1671,7 +1695,7 @@ private void onConnectionChanged(MoneroRpcConnection connection) { if (StringUtils.equals(oldProxyUri, newProxyUri)) { wallet.setDaemonConnection(connection); } else { - log.info("Restarting main wallet because proxy URI has changed, old={}, new={}", oldProxyUri, newProxyUri); + log.info("Restarting main wallet because proxy URI has changed, old={}, new={}", oldProxyUri, newProxyUri); // TODO: set proxy without restarting wallet closeMainWallet(true); maybeInitMainWallet(false); } @@ -1680,7 +1704,13 @@ private void onConnectionChanged(MoneroRpcConnection connection) { wallet.setProxyUri(connection.getProxyUri()); } - // sync wallet on new thread + // switch if wallet disconnected + if (Boolean.TRUE.equals(connection.isConnected() && !wallet.isConnectedToDaemon())) { + log.warn("Switching to next best connection because main wallet is disconnected"); + if (xmrConnectionService.requestSwitchToNextBestConnection()) return; // calls back into this method + } + + // update poll period if (connection != null && !isShutDownStarted) { wallet.getDaemonConnection().setPrintStackTrace(PRINT_RPC_STACK_TRACE); updatePollPeriod(); @@ -1817,7 +1847,7 @@ private void doPollWallet(boolean updateTxs) { // switch to best connection if wallet is too far behind if (wasWalletSynced && walletHeight.get() < xmrConnectionService.getTargetHeight() - NUM_BLOCKS_BEHIND_TOLERANCE && !Config.baseCurrencyNetwork().isTestnet()) { log.warn("Updating connection because main wallet is {} blocks behind monerod, wallet height={}, monerod height={}", xmrConnectionService.getTargetHeight() - walletHeight.get(), walletHeight.get(), lastInfo.getHeight()); - xmrConnectionService.switchToBestConnection(); + if (xmrConnectionService.isConnected()) xmrConnectionService.requestSwitchToNextBestConnection(); } // sync wallet if behind daemon diff --git a/desktop/src/main/java/haveno/desktop/main/funds/withdrawal/WithdrawalView.java b/desktop/src/main/java/haveno/desktop/main/funds/withdrawal/WithdrawalView.java index 44a66881e32..463c310a0a7 100644 --- a/desktop/src/main/java/haveno/desktop/main/funds/withdrawal/WithdrawalView.java +++ b/desktop/src/main/java/haveno/desktop/main/funds/withdrawal/WithdrawalView.java @@ -270,6 +270,7 @@ private void onWithdraw() { if (isNotEnoughMoney(e.getMessage())) throw e; log.warn("Error creating creating withdraw tx, attempt={}/{}, error={}", i + 1, TradeProtocol.MAX_ATTEMPTS, e.getMessage()); if (i == TradeProtocol.MAX_ATTEMPTS - 1) throw e; + if (xmrWalletService.getConnectionService().isConnected()) xmrWalletService.getConnectionService().requestSwitchToNextBestConnection(); HavenoUtils.waitFor(TradeProtocol.REPROCESS_DELAY_MS); // wait before retrying } }