diff --git a/common/src/main/java/bisq/common/app/Capabilities.java b/common/src/main/java/bisq/common/app/Capabilities.java index 25913c9b0a7..d0b3e50a3f8 100644 --- a/common/src/main/java/bisq/common/app/Capabilities.java +++ b/common/src/main/java/bisq/common/app/Capabilities.java @@ -98,6 +98,10 @@ public boolean containsAll(Capability... capabilities) { return this.capabilities.containsAll(Arrays.asList(capabilities)); } + public boolean contains(Capability capability) { + return this.capabilities.contains(capability); + } + public boolean isEmpty() { return capabilities.isEmpty(); } @@ -180,4 +184,18 @@ public String prettyPrint() { public int size() { return capabilities.size(); } + + // We return true if our capabilities have less capabilities than the parameter value + public boolean hasLess(Capabilities other) { + return findHighestCapability(this) < findHighestCapability(other); + } + + // We use the sum of all capabilities. Alternatively we could use the highest entry. + // Neither would support removal of past capabilities, a use case we never had so far and which might have + // backward compatibility issues, so we should treat capabilities as an append-only data structure. + public int findHighestCapability(Capabilities capabilities) { + return (int) capabilities.capabilities.stream() + .mapToLong(e -> (long) e.ordinal()) + .sum(); + } } diff --git a/common/src/main/java/bisq/common/app/Capability.java b/common/src/main/java/bisq/common/app/Capability.java index 33040bbfa40..1c9aebd8898 100644 --- a/common/src/main/java/bisq/common/app/Capability.java +++ b/common/src/main/java/bisq/common/app/Capability.java @@ -40,5 +40,7 @@ public enum Capability { SIGNED_ACCOUNT_AGE_WITNESS, // Supports the signed account age witness feature MEDIATION, // Supports mediation feature REFUND_AGENT, // Supports refund agents - TRADE_STATISTICS_HASH_UPDATE // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. + TRADE_STATISTICS_HASH_UPDATE, // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. + NO_ADDRESS_PRE_FIX, // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix. + TRADE_STATISTICS_3 // We used a new reduced trade statistics model from v1.4.0 on } diff --git a/common/src/main/java/bisq/common/app/Version.java b/common/src/main/java/bisq/common/app/Version.java index fd8cfb64634..45fc057242b 100644 --- a/common/src/main/java/bisq/common/app/Version.java +++ b/common/src/main/java/bisq/common/app/Version.java @@ -128,7 +128,6 @@ public static void printVersion() { '}'); } - //TODO move to consensus area public static final byte COMPENSATION_REQUEST = (byte) 0x01; public static final byte REIMBURSEMENT_REQUEST = (byte) 0x01; public static final byte PROPOSAL = (byte) 0x01; diff --git a/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java b/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java index 6aa722a4d7b..94105d34f39 100644 --- a/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java +++ b/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java @@ -24,8 +24,5 @@ * Better to use the excludeFromJsonDataMap (annotated with @JsonExclude; used in PaymentAccountPayload) to * add a key/value pair. */ -// TODO PubKeyRing and NodeAddress (network) are using UsedForTradeContractJson that is why it is in common module, -// which is a bit weird... Maybe we need either rename common or split it to util and common where common is common code -// used in network and core? public interface UsedForTradeContractJson { } diff --git a/common/src/main/java/bisq/common/crypto/Encryption.java b/common/src/main/java/bisq/common/crypto/Encryption.java index f4e1f8e0221..8be207fe4c2 100644 --- a/common/src/main/java/bisq/common/crypto/Encryption.java +++ b/common/src/main/java/bisq/common/crypto/Encryption.java @@ -48,7 +48,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -// TODO is Hmac needed/make sense? public class Encryption { private static final Logger log = LoggerFactory.getLogger(Encryption.class); diff --git a/common/src/main/java/bisq/common/crypto/KeyStorage.java b/common/src/main/java/bisq/common/crypto/KeyStorage.java index c132ab0042f..7b4ecece087 100644 --- a/common/src/main/java/bisq/common/crypto/KeyStorage.java +++ b/common/src/main/java/bisq/common/crypto/KeyStorage.java @@ -55,7 +55,6 @@ import static bisq.common.util.Preconditions.checkDir; -// TODO: use a password protection for key storage @Singleton public class KeyStorage { private static final Logger log = LoggerFactory.getLogger(KeyStorage.class); diff --git a/common/src/main/java/bisq/common/file/JsonFileManager.java b/common/src/main/java/bisq/common/file/JsonFileManager.java index 9cbc91f7b29..1d1db8a9d88 100644 --- a/common/src/main/java/bisq/common/file/JsonFileManager.java +++ b/common/src/main/java/bisq/common/file/JsonFileManager.java @@ -30,7 +30,7 @@ @Slf4j public class JsonFileManager { - private final ThreadPoolExecutor executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60); + private final ThreadPoolExecutor executor; private final File dir; @@ -41,6 +41,8 @@ public class JsonFileManager { public JsonFileManager(File dir) { this.dir = dir; + this.executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60); + if (!dir.exists()) if (!dir.mkdir()) log.warn("make dir failed"); diff --git a/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java b/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java index 535d65d0486..330fbd72161 100644 --- a/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java +++ b/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java @@ -48,7 +48,6 @@ public Message toProtoMessage() { return getNetworkEnvelopeBuilder().build(); } - // todo remove public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { return getNetworkEnvelopeBuilder().build(); } diff --git a/common/src/test/java/bisq/common/app/CapabilitiesTest.java b/common/src/test/java/bisq/common/app/CapabilitiesTest.java index 735fc899eaf..f939ff9cd83 100644 --- a/common/src/test/java/bisq/common/app/CapabilitiesTest.java +++ b/common/src/test/java/bisq/common/app/CapabilitiesTest.java @@ -23,6 +23,7 @@ import org.junit.Test; +import static bisq.common.app.Capability.DAO_FULL_NODE; import static bisq.common.app.Capability.SEED_NODE; import static bisq.common.app.Capability.TRADE_STATISTICS; import static bisq.common.app.Capability.TRADE_STATISTICS_2; @@ -40,6 +41,47 @@ public void testNoCapabilitiesAvailable() { assertFalse(DUT.containsAll(new Capabilities(SEED_NODE))); } + @Test + public void testHasLess() { + assertTrue(new Capabilities().hasLess(new Capabilities(SEED_NODE))); + assertFalse(new Capabilities().hasLess(new Capabilities())); + assertFalse(new Capabilities(SEED_NODE).hasLess(new Capabilities())); + assertTrue(new Capabilities(SEED_NODE).hasLess(new Capabilities(DAO_FULL_NODE))); + assertFalse(new Capabilities(DAO_FULL_NODE).hasLess(new Capabilities(SEED_NODE))); + + Capabilities all = new Capabilities( + TRADE_STATISTICS, + TRADE_STATISTICS_2, + Capability.ACCOUNT_AGE_WITNESS, + Capability.ACK_MSG, + Capability.PROPOSAL, + Capability.BLIND_VOTE, + Capability.DAO_STATE, + Capability.BUNDLE_OF_ENVELOPES, + Capability.MEDIATION, + Capability.SIGNED_ACCOUNT_AGE_WITNESS, + Capability.REFUND_AGENT, + Capability.TRADE_STATISTICS_HASH_UPDATE + ); + Capabilities other = new Capabilities( + TRADE_STATISTICS, + TRADE_STATISTICS_2, + Capability.ACCOUNT_AGE_WITNESS, + Capability.ACK_MSG, + Capability.PROPOSAL, + Capability.BLIND_VOTE, + Capability.DAO_STATE, + Capability.BUNDLE_OF_ENVELOPES, + Capability.MEDIATION, + Capability.SIGNED_ACCOUNT_AGE_WITNESS, + Capability.REFUND_AGENT, + Capability.TRADE_STATISTICS_HASH_UPDATE, + Capability.NO_ADDRESS_PRE_FIX + ); + + assertTrue(all.hasLess(other)); + } + @Test public void testO() { Capabilities DUT = new Capabilities(TRADE_STATISTICS); diff --git a/core/src/main/java/bisq/core/api/CoreApi.java b/core/src/main/java/bisq/core/api/CoreApi.java index e944265a4b7..be694bc7130 100644 --- a/core/src/main/java/bisq/core/api/CoreApi.java +++ b/core/src/main/java/bisq/core/api/CoreApi.java @@ -22,7 +22,7 @@ import bisq.core.offer.Offer; import bisq.core.offer.OfferPayload; import bisq.core.payment.PaymentAccount; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.common.app.Version; @@ -196,7 +196,7 @@ public void removeWalletPassword(String password) { walletsService.removeWalletPassword(password); } - public List getTradeStatistics() { + public List getTradeStatistics() { return new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet()); } diff --git a/core/src/main/java/bisq/core/app/BisqExecutable.java b/core/src/main/java/bisq/core/app/BisqExecutable.java index 8e403b97dc1..66c03dede43 100644 --- a/core/src/main/java/bisq/core/app/BisqExecutable.java +++ b/core/src/main/java/bisq/core/app/BisqExecutable.java @@ -25,6 +25,7 @@ import bisq.core.setup.CorePersistedDataHost; import bisq.core.setup.CoreSetup; import bisq.core.support.dispute.arbitration.arbitrator.ArbitratorManager; +import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.trade.txproof.xmr.XmrTxProofService; import bisq.network.p2p.P2PService; @@ -220,6 +221,7 @@ public void gracefulShutDown(ResultHandler resultHandler) { try { injector.getInstance(ArbitratorManager.class).shutDown(); + injector.getInstance(TradeStatisticsManager.class).shutDown(); injector.getInstance(XmrTxProofService.class).shutDown(); injector.getInstance(DaoSetup.class).shutDown(); injector.getInstance(AvoidStandbyModeService.class).shutDown(); diff --git a/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java b/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java index f66614ec855..cc902185057 100644 --- a/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java +++ b/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java @@ -154,10 +154,10 @@ public void updateAssetStates() { // TradeAmountDateTuple object holding only the data we need. Map> lookupMap = new HashMap<>(); tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> CurrencyUtil.isCryptoCurrency(e.getBaseCurrency())) + .filter(e -> CurrencyUtil.isCryptoCurrency(e.getCurrency())) .forEach(e -> { - lookupMap.putIfAbsent(e.getBaseCurrency(), new ArrayList<>()); - lookupMap.get(e.getBaseCurrency()).add(new TradeAmountDateTuple(e.getTradeAmount().getValue(), e.getTradeDate().getTime())); + lookupMap.putIfAbsent(e.getCurrency(), new ArrayList<>()); + lookupMap.get(e.getCurrency()).add(new TradeAmountDateTuple(e.getAmount(), e.getDate())); }); getStatefulAssets().stream() diff --git a/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java b/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java index 19192e543d0..ea73a102c26 100644 --- a/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java +++ b/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java @@ -170,7 +170,7 @@ private void connectToNextNode() { private void connectToAnyFullNode() { Capabilities required = new Capabilities(Capability.DAO_FULL_NODE); - List list = peerManager.getLivePeers(null).stream() + List list = peerManager.getLivePeers().stream() .filter(peer -> peer.getCapabilities().containsAll(required)) .collect(Collectors.toList()); diff --git a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java index 5c7a861a7c4..0b40e80b96c 100644 --- a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java +++ b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java @@ -180,7 +180,7 @@ public void onConnection(Connection connection) { public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { closeHandler(connection); - if (peerManager.isNodeBanned(closeConnectionReason, connection)) { + if (peerManager.isPeerBanned(closeConnectionReason, connection)) { connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { seedNodeAddresses.remove(nodeAddress); removeFromRequestBlocksHandlerMap(nodeAddress); diff --git a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java index 478038ad9ab..8745bd3c565 100644 --- a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java +++ b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java @@ -19,7 +19,7 @@ import bisq.core.support.dispute.agent.DisputeAgent; import bisq.core.support.dispute.agent.DisputeAgentManager; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.common.util.Tuple2; @@ -42,36 +42,37 @@ @Slf4j public class DisputeAgentSelection { + public static final int LOOK_BACK_RANGE = 100; + public static T getLeastUsedMediator(TradeStatisticsManager tradeStatisticsManager, DisputeAgentManager disputeAgentManager) { return getLeastUsedDisputeAgent(tradeStatisticsManager, disputeAgentManager, - TradeStatistics2.MEDIATOR_ADDRESS); + true); } public static T getLeastUsedRefundAgent(TradeStatisticsManager tradeStatisticsManager, DisputeAgentManager disputeAgentManager) { return getLeastUsedDisputeAgent(tradeStatisticsManager, disputeAgentManager, - TradeStatistics2.REFUND_AGENT_ADDRESS); + false); } private static T getLeastUsedDisputeAgent(TradeStatisticsManager tradeStatisticsManager, DisputeAgentManager disputeAgentManager, - String extraMapKey) { + boolean isMediator) { // We take last 100 entries from trade statistics - List list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet()); - list.sort(Comparator.comparing(TradeStatistics2::getTradeDate)); + List list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet()); + list.sort(Comparator.comparing(TradeStatistics3::getDate)); Collections.reverse(list); if (!list.isEmpty()) { - int max = Math.min(list.size(), 100); + int max = Math.min(list.size(), LOOK_BACK_RANGE); list = list.subList(0, max); } // We stored only first 4 chars of disputeAgents onion address List lastAddressesUsedInTrades = list.stream() - .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap() != null) - .map(tradeStatistics2 -> tradeStatistics2.getExtraDataMap().get(extraMapKey)) + .map(tradeStatistics3 -> isMediator ? tradeStatistics3.getMediator() : tradeStatistics3.getRefundAgent()) .filter(Objects::nonNull) .collect(Collectors.toList()); diff --git a/core/src/main/java/bisq/core/proto/CoreProtoResolver.java b/core/src/main/java/bisq/core/proto/CoreProtoResolver.java index 999f4e8e5f1..921a1fa5a4a 100644 --- a/core/src/main/java/bisq/core/proto/CoreProtoResolver.java +++ b/core/src/main/java/bisq/core/proto/CoreProtoResolver.java @@ -54,6 +54,7 @@ import bisq.core.payment.payload.WeChatPayAccountPayload; import bisq.core.payment.payload.WesternUnionAccountPayload; import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.common.proto.ProtoResolver; import bisq.common.proto.ProtobufferRuntimeException; @@ -178,6 +179,8 @@ public PersistablePayload fromProto(protobuf.PersistableNetworkPayload proto) { return BlindVotePayload.fromProto(proto.getBlindVotePayload()); case SIGNED_WITNESS: return SignedWitness.fromProto(proto.getSignedWitness()); + case TRADE_STATISTICS3: + return TradeStatistics3.fromProto(proto.getTradeStatistics3()); default: throw new ProtobufferRuntimeException("Unknown proto message case (PB.PersistableNetworkPayload). messageCase=" + proto.getMessageCase()); } diff --git a/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java b/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java index 8d57a75c14b..675011cec3d 100644 --- a/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java +++ b/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java @@ -59,7 +59,6 @@ import bisq.core.trade.messages.PeerPublishedDelayedPayoutTxMessage; import bisq.core.trade.messages.RefreshTradeStateRequest; import bisq.core.trade.messages.TraderSignedWitnessMessage; -import bisq.core.trade.statistics.TradeStatistics; import bisq.network.p2p.AckMessage; import bisq.network.p2p.BundleOfEnvelopes; @@ -265,9 +264,6 @@ public NetworkPayload fromProto(protobuf.StoragePayload proto) { return RefundAgent.fromProto(proto.getRefundAgent()); case FILTER: return Filter.fromProto(proto.getFilter()); - case TRADE_STATISTICS: - // Still used to convert TradeStatistics data from pre v0.6 versions - return TradeStatistics.fromProto(proto.getTradeStatistics()); case MAILBOX_STORAGE_PAYLOAD: return MailboxStoragePayload.fromProto(proto.getMailboxStoragePayload()); case OFFER_PAYLOAD: diff --git a/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java b/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java index efefffe881a..26addc24b0e 100644 --- a/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java +++ b/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java @@ -39,6 +39,7 @@ import bisq.core.support.dispute.refund.RefundDisputeList; import bisq.core.trade.TradableList; import bisq.core.trade.statistics.TradeStatistics2Store; +import bisq.core.trade.statistics.TradeStatistics3Store; import bisq.core.user.PreferencesPayload; import bisq.core.user.UserPayload; @@ -126,6 +127,8 @@ public PersistableEnvelope fromProto(protobuf.PersistableEnvelope proto) { return UnconfirmedBsqChangeOutputList.fromProto(proto.getUnconfirmedBsqChangeOutputList()); case SIGNED_WITNESS_STORE: return SignedWitnessStore.fromProto(proto.getSignedWitnessStore()); + case TRADE_STATISTICS3_STORE: + return TradeStatistics3Store.fromProto(proto.getTradeStatistics3Store()); default: throw new ProtobufferRuntimeException("Unknown proto message case(PB.PersistableEnvelope). " + diff --git a/core/src/main/java/bisq/core/provider/fee/FeeService.java b/core/src/main/java/bisq/core/provider/fee/FeeService.java index 76427f5b3aa..b43c4491b62 100644 --- a/core/src/main/java/bisq/core/provider/fee/FeeService.java +++ b/core/src/main/java/bisq/core/provider/fee/FeeService.java @@ -52,7 +52,6 @@ import static com.google.common.base.Preconditions.checkNotNull; -// TODO use dao parameters for fee @Slf4j public class FeeService { diff --git a/core/src/main/java/bisq/core/provider/price/PriceFeedService.java b/core/src/main/java/bisq/core/provider/price/PriceFeedService.java index 3db0e855e09..b90b9128507 100644 --- a/core/src/main/java/bisq/core/provider/price/PriceFeedService.java +++ b/core/src/main/java/bisq/core/provider/price/PriceFeedService.java @@ -23,7 +23,7 @@ import bisq.core.monetary.Price; import bisq.core.provider.PriceNodeHttpClient; import bisq.core.provider.ProvidersRepository; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.user.Preferences; import bisq.network.http.HttpClient; @@ -50,6 +50,7 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -288,12 +289,12 @@ public Date getLastRequestTimeStamp() { return new Date(epochInMillisAtLastRequest); } - public void applyLatestBisqMarketPrice(Set tradeStatisticsSet) { + public void applyLatestBisqMarketPrice(Set tradeStatisticsSet) { // takes about 10 ms for 5000 items - Map> mapByCurrencyCode = new HashMap<>(); + Map> mapByCurrencyCode = new HashMap<>(); tradeStatisticsSet.forEach(e -> { - final List list; - final String currencyCode = e.getCurrencyCode(); + List list; + String currencyCode = e.getCurrency(); if (mapByCurrencyCode.containsKey(currencyCode)) { list = mapByCurrencyCode.get(currencyCode); } else { @@ -306,9 +307,9 @@ public void applyLatestBisqMarketPrice(Set tradeStatisticsSet) mapByCurrencyCode.values().stream() .filter(list -> !list.isEmpty()) .forEach(list -> { - list.sort((o1, o2) -> o1.getTradeDate().compareTo(o2.getTradeDate())); - TradeStatistics2 tradeStatistics = list.get(list.size() - 1); - setBisqMarketPrice(tradeStatistics.getCurrencyCode(), tradeStatistics.getTradePrice()); + list.sort(Comparator.comparing(TradeStatistics3::getTradeDate)); + TradeStatistics3 tradeStatistics = list.get(list.size() - 1); + setBisqMarketPrice(tradeStatistics.getCurrency(), tradeStatistics.getTradePrice()); }); } diff --git a/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java b/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java index fcd03280337..c5c70e8877d 100644 --- a/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java +++ b/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java @@ -39,7 +39,9 @@ static void setSupportedCapabilities(Config config) { Capability.MEDIATION, Capability.SIGNED_ACCOUNT_AGE_WITNESS, Capability.REFUND_AGENT, - Capability.TRADE_STATISTICS_HASH_UPDATE + Capability.TRADE_STATISTICS_HASH_UPDATE, + Capability.NO_ADDRESS_PRE_FIX, + Capability.TRADE_STATISTICS_3 ); if (config.daoActivated) { diff --git a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java index ac6480bb1b2..73a73110d4c 100644 --- a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java +++ b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java @@ -214,8 +214,6 @@ public void updateMap() { observableMap.putAll(filtered); observableMap.values().forEach(this::addAcceptedDisputeAgentToUser); - - log.info("Available disputeAgents: {}", observableMap.keySet()); } diff --git a/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java b/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java index 8552003199d..29f6b6939aa 100644 --- a/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java +++ b/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java @@ -157,7 +157,7 @@ public void cleanupDisputes() { @Override protected String getDisputeInfo(Dispute dispute) { String role = Res.get("shared.arbitrator").toLowerCase(); - String link = "https://docs.bisq.network/trading-rules.html#legacy-arbitration"; //TODO needs to be created + String link = "https://docs.bisq.network/trading-rules.html#legacy-arbitration"; return Res.get("support.initialInfo", role, role, link); } diff --git a/core/src/main/java/bisq/core/trade/TradeModule.java b/core/src/main/java/bisq/core/trade/TradeModule.java index 779cf6e7600..751bd5a0eb1 100644 --- a/core/src/main/java/bisq/core/trade/TradeModule.java +++ b/core/src/main/java/bisq/core/trade/TradeModule.java @@ -24,8 +24,6 @@ import bisq.core.trade.closed.ClosedTradableManager; import bisq.core.trade.failed.FailedTradesManager; import bisq.core.trade.statistics.ReferralIdService; -import bisq.core.trade.statistics.TradeStatistics2StorageService; -import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.common.app.AppModule; import bisq.common.config.Config; @@ -46,8 +44,6 @@ public TradeModule(Config config) { @Override protected void configure() { bind(TradeManager.class).in(Singleton.class); - bind(TradeStatisticsManager.class).in(Singleton.class); - bind(TradeStatistics2StorageService.class).in(Singleton.class); bind(ClosedTradableManager.class).in(Singleton.class); bind(FailedTradesManager.class).in(Singleton.class); bind(AccountAgeWitnessService.class).in(Singleton.class); diff --git a/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java b/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java index 64c718ce412..ffff35a79bb 100644 --- a/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java @@ -24,7 +24,6 @@ import bisq.core.trade.messages.PayoutTxPublishedMessage; import bisq.core.trade.messages.TradeMessage; import bisq.core.trade.protocol.tasks.ApplyFilter; -import bisq.core.trade.protocol.tasks.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.TradeTask; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessDepositTxAndDelayedPayoutTxMessage; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessPayoutTxPublishedMessage; @@ -114,8 +113,7 @@ protected void handle(DepositTxAndDelayedPayoutTxMessage message, NodeAddress pe removeMailboxMessageAfterProcessing(message); })) .setup(tasks(BuyerProcessDepositTxAndDelayedPayoutTxMessage.class, - BuyerVerifiesFinalDelayedPayoutTx.class, - PublishTradeStatistics.class) + BuyerVerifiesFinalDelayedPayoutTx.class) .using(new TradeTaskRunner(trade, () -> { stopTimeout(); diff --git a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java index 35eea700b6c..9f1acab3689 100644 --- a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java @@ -23,13 +23,13 @@ import bisq.core.trade.messages.DelayedPayoutTxSignatureResponse; import bisq.core.trade.messages.TradeMessage; import bisq.core.trade.protocol.tasks.ApplyFilter; -import bisq.core.trade.protocol.tasks.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.TradeTask; import bisq.core.trade.protocol.tasks.seller.SellerBroadcastPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerFinalizesDelayedPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerProcessCounterCurrencyTransferStartedMessage; import bisq.core.trade.protocol.tasks.seller.SellerProcessDelayedPayoutTxSignatureResponse; import bisq.core.trade.protocol.tasks.seller.SellerPublishesDepositTx; +import bisq.core.trade.protocol.tasks.seller.SellerPublishesTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerSendPayoutTxPublishedMessage; import bisq.core.trade.protocol.tasks.seller.SellerSendsDepositTxAndDelayedPayoutTxMessage; import bisq.core.trade.protocol.tasks.seller.SellerSignAndFinalizePayoutTx; @@ -39,12 +39,17 @@ import bisq.common.handlers.ErrorMessageHandler; import bisq.common.handlers.ResultHandler; +import bisq.common.util.Utilities; + +import java.util.Date; +import java.util.GregorianCalendar; import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class SellerProtocol extends DisputeProtocol { enum SellerEvent implements FluentProtocol.Event { + STARTUP, PAYMENT_RECEIVED } @@ -52,6 +57,27 @@ public SellerProtocol(SellerTrade trade) { super(trade); } + @Override + protected void onInitialized() { + super.onInitialized(); + + // We get called the constructor with any possible state and phase. As we don't want to log an error for such + // cases we use the alternative 'given' method instead of 'expect'. + + // We only re-publish for about 2 weeks after 1.4.0 release until most nodes have updated to + // achieve sufficient resilience. + boolean currentDateBeforeCutOffDate = new Date().before(Utilities.getUTCDate(2020, GregorianCalendar.NOVEMBER, 1)); + given(anyPhase(Trade.Phase.DEPOSIT_PUBLISHED, + Trade.Phase.DEPOSIT_CONFIRMED, + Trade.Phase.FIAT_SENT, + Trade.Phase.FIAT_RECEIVED, + Trade.Phase.PAYOUT_PUBLISHED) + .with(SellerEvent.STARTUP) + .preCondition(currentDateBeforeCutOffDate)) + .setup(tasks(SellerPublishesTradeStatistics.class)) + .executeTasks(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Mailbox @@ -79,7 +105,7 @@ protected void handle(DelayedPayoutTxSignatureResponse message, NodeAddress peer SellerFinalizesDelayedPayoutTx.class, SellerSendsDepositTxAndDelayedPayoutTxMessage.class, SellerPublishesDepositTx.class, - PublishTradeStatistics.class)) + SellerPublishesTradeStatistics.class)) .run(() -> { // We stop timeout here and don't start a new one as the // SellerSendsDepositTxAndDelayedPayoutTxMessage repeats the send the message and has it's own diff --git a/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java b/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java index f27fa0804af..af259e25c74 100644 --- a/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java @@ -30,6 +30,7 @@ import bisq.network.p2p.DecryptedMessageWithPubKey; import bisq.network.p2p.MailboxMessage; import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.P2PService; import bisq.network.p2p.SendMailboxMessageListener; import bisq.network.p2p.messaging.DecryptedMailboxListener; @@ -77,8 +78,8 @@ protected void onInitialized() { processModel.getP2PService().addDecryptedDirectMessageListener(this); } processModel.getP2PService().addDecryptedMailboxListener(this); - processModel.getP2PService().getMailboxMap().values() - .stream().map(e -> e.second) + processModel.getP2PService().getMailboxItemsByUid().values() + .stream().map(P2PService.MailboxItem::getDecryptedMessageWithPubKey) .forEach(this::handleDecryptedMessageWithPubKey); } diff --git a/core/src/main/java/bisq/core/trade/protocol/tasks/PublishTradeStatistics.java b/core/src/main/java/bisq/core/trade/protocol/tasks/PublishTradeStatistics.java deleted file mode 100644 index 8eb71ffdd87..00000000000 --- a/core/src/main/java/bisq/core/trade/protocol/tasks/PublishTradeStatistics.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.protocol.tasks; - -import bisq.core.offer.Offer; -import bisq.core.offer.OfferPayload; -import bisq.core.trade.Trade; -import bisq.core.trade.statistics.TradeStatistics2; - -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.network.TorNetworkNode; - -import bisq.common.taskrunner.TaskRunner; - -import java.util.HashMap; -import java.util.Map; - -import lombok.extern.slf4j.Slf4j; - -import static com.google.common.base.Preconditions.checkNotNull; - -@Slf4j -public class PublishTradeStatistics extends TradeTask { - public PublishTradeStatistics(TaskRunner taskHandler, Trade trade) { - super(taskHandler, trade); - } - - @Override - protected void run() { - try { - runInterceptHook(); - - checkNotNull(trade.getDepositTx()); - - Map extraDataMap = new HashMap<>(); - if (processModel.getReferralIdService().getOptionalReferralId().isPresent()) { - extraDataMap.put(OfferPayload.REFERRAL_ID, processModel.getReferralIdService().getOptionalReferralId().get()); - } - - NodeAddress mediatorNodeAddress = checkNotNull(trade.getMediatorNodeAddress()); - // The first 4 chars are sufficient to identify a mediator. - // For testing with regtest/localhost we use the full address as its localhost and would result in - // same values for multiple mediators. - NetworkNode networkNode = model.getProcessModel().getP2PService().getNetworkNode(); - String address = networkNode instanceof TorNetworkNode ? - mediatorNodeAddress.getFullAddress().substring(0, 4) : - mediatorNodeAddress.getFullAddress(); - extraDataMap.put(TradeStatistics2.MEDIATOR_ADDRESS, address); - - Offer offer = checkNotNull(trade.getOffer()); - TradeStatistics2 tradeStatistics = new TradeStatistics2(offer.getOfferPayload(), - trade.getTradePrice(), - checkNotNull(trade.getTradeAmount()), - trade.getDate(), - trade.getDepositTxId(), - extraDataMap); - processModel.getP2PService().addPersistableNetworkPayload(tradeStatistics, true); - - complete(); - } catch (Throwable t) { - failed(t); - } - } -} diff --git a/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java new file mode 100644 index 00000000000..0489ac0e325 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java @@ -0,0 +1,105 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.core.trade.protocol.tasks.seller; + +import bisq.core.offer.Offer; +import bisq.core.offer.OfferPayload; +import bisq.core.trade.Trade; +import bisq.core.trade.protocol.tasks.TradeTask; +import bisq.core.trade.statistics.TradeStatistics3; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.network.TorNetworkNode; + +import bisq.common.app.Capability; +import bisq.common.taskrunner.TaskRunner; + +import java.util.HashMap; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +import static com.google.common.base.Preconditions.checkNotNull; + +@Slf4j +public class SellerPublishesTradeStatistics extends TradeTask { + public SellerPublishesTradeStatistics(TaskRunner taskHandler, Trade trade) { + super(taskHandler, trade); + } + + @Override + protected void run() { + try { + runInterceptHook(); + + checkNotNull(trade.getDepositTx()); + + processModel.getP2PService().findPeersCapabilities(trade.getTradingPeerNodeAddress()) + .filter(capabilities -> capabilities.containsAll(Capability.TRADE_STATISTICS_3)) + .ifPresentOrElse(capabilities -> { + // Our peer has updated, so as we are the seller we will publish the trade statistics. + // The peer as buyer does not publish anymore with v.1.4.0 (where Capability.TRADE_STATISTICS_3 was added) + + Map extraDataMap = new HashMap<>(); + if (processModel.getReferralIdService().getOptionalReferralId().isPresent()) { + extraDataMap.put(OfferPayload.REFERRAL_ID, processModel.getReferralIdService().getOptionalReferralId().get()); + } + + NodeAddress mediatorNodeAddress = checkNotNull(trade.getMediatorNodeAddress()); + // The first 4 chars are sufficient to identify a mediator. + // For testing with regtest/localhost we use the full address as its localhost and would result in + // same values for multiple mediators. + NetworkNode networkNode = model.getProcessModel().getP2PService().getNetworkNode(); + String truncatedMediatorNodeAddress = networkNode instanceof TorNetworkNode ? + mediatorNodeAddress.getFullAddress().substring(0, 4) : + mediatorNodeAddress.getFullAddress(); + + NodeAddress refundAgentNodeAddress = checkNotNull(trade.getRefundAgentNodeAddress()); + String truncatedRefundAgentNodeAddress = networkNode instanceof TorNetworkNode ? + refundAgentNodeAddress.getFullAddress().substring(0, 4) : + refundAgentNodeAddress.getFullAddress(); + + Offer offer = checkNotNull(trade.getOffer()); + TradeStatistics3 tradeStatistics = new TradeStatistics3(offer.getCurrencyCode(), + trade.getTradePrice().getValue(), + trade.getTradeAmountAsLong(), + offer.getPaymentMethod().getId(), + trade.getTakeOfferDate().getTime(), + truncatedMediatorNodeAddress, + truncatedRefundAgentNodeAddress, + extraDataMap); + if (tradeStatistics.isValid()) { + log.info("Publishing trade statistics"); + processModel.getP2PService().addPersistableNetworkPayload(tradeStatistics, true); + } else { + log.warn("Trade statistics are invalid. We do not publish. {}", tradeStatistics); + } + + complete(); + }, + () -> { + log.info("Our peer does not has updated yet, so they will publish the trade statistics. " + + "To avoid duplicates we do not publish from our side."); + complete(); + }); + } catch (Throwable t) { + failed(t); + } + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java deleted file mode 100644 index ad543213191..00000000000 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import bisq.core.monetary.Altcoin; -import bisq.core.monetary.AltcoinExchangeRate; -import bisq.core.monetary.Price; -import bisq.core.monetary.Volume; -import bisq.core.offer.OfferPayload; - -import bisq.network.p2p.storage.payload.ExpirablePayload; -import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import bisq.common.crypto.Sig; -import bisq.common.proto.persistable.PersistablePayload; -import bisq.common.util.CollectionUtils; -import bisq.common.util.ExtraDataMapValidator; -import bisq.common.util.JsonExclude; - -import com.google.protobuf.ByteString; - -import org.bitcoinj.core.Coin; -import org.bitcoinj.utils.ExchangeRate; -import org.bitcoinj.utils.Fiat; - -import java.security.PublicKey; - -import java.util.Date; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import lombok.EqualsAndHashCode; -import lombok.Value; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nullable; - -/** - * @deprecated Was used in pre v0.6.0 version - */ -@Deprecated -@Slf4j -@EqualsAndHashCode(exclude = {"signaturePubKeyBytes"}) -@Value -public final class TradeStatistics implements ProcessOncePersistableNetworkPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { - private final OfferPayload.Direction direction; - private final String baseCurrency; - private final String counterCurrency; - private final String offerPaymentMethod; - private final long offerDate; - private final boolean offerUseMarketBasedPrice; - private final double offerMarketPriceMargin; - private final long offerAmount; - private final long offerMinAmount; - private final String offerId; - private final long tradePrice; - private final long tradeAmount; - private final long tradeDate; - private final String depositTxId; - @JsonExclude - private final byte[] signaturePubKeyBytes; - @JsonExclude - transient private final PublicKey signaturePubKey; - - // Should be only used in emergency case if we need to add data but do not want to break backward compatibility - // at the P2P network storage checks. The hash of the object will be used to verify if the data is valid. Any new - // field in a class would break that hash and therefore break the storage mechanism. - @Nullable - private Map extraDataMap; - - public TradeStatistics(OfferPayload offerPayload, - Price tradePrice, - Coin tradeAmount, - Date tradeDate, - String depositTxId, - byte[] signaturePubKeyBytes) { - this(offerPayload.getDirection(), - offerPayload.getBaseCurrencyCode(), - offerPayload.getCounterCurrencyCode(), - offerPayload.getPaymentMethodId(), - offerPayload.getDate(), - offerPayload.isUseMarketBasedPrice(), - offerPayload.getMarketPriceMargin(), - offerPayload.getAmount(), - offerPayload.getMinAmount(), - offerPayload.getId(), - tradePrice.getValue(), - tradeAmount.value, - tradeDate.getTime(), - depositTxId, - signaturePubKeyBytes, - null); - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // PROTO BUFFER - /////////////////////////////////////////////////////////////////////////////////////////// - - TradeStatistics(OfferPayload.Direction direction, - String baseCurrency, - String counterCurrency, - String offerPaymentMethod, - long offerDate, - boolean offerUseMarketBasedPrice, - double offerMarketPriceMargin, - long offerAmount, - long offerMinAmount, - String offerId, - long tradePrice, - long tradeAmount, - long tradeDate, - String depositTxId, - byte[] signaturePubKeyBytes, - @Nullable Map extraDataMap) { - this.direction = direction; - this.baseCurrency = baseCurrency; - this.counterCurrency = counterCurrency; - this.offerPaymentMethod = offerPaymentMethod; - this.offerDate = offerDate; - this.offerUseMarketBasedPrice = offerUseMarketBasedPrice; - this.offerMarketPriceMargin = offerMarketPriceMargin; - this.offerAmount = offerAmount; - this.offerMinAmount = offerMinAmount; - this.offerId = offerId; - this.tradePrice = tradePrice; - this.tradeAmount = tradeAmount; - this.tradeDate = tradeDate; - this.depositTxId = depositTxId; - this.signaturePubKeyBytes = signaturePubKeyBytes; - this.extraDataMap = ExtraDataMapValidator.getValidatedExtraDataMap(extraDataMap); - - signaturePubKey = Sig.getPublicKeyFromBytes(signaturePubKeyBytes); - } - - @Override - public protobuf.StoragePayload toProtoMessage() { - final protobuf.TradeStatistics.Builder builder = protobuf.TradeStatistics.newBuilder() - .setDirection(OfferPayload.Direction.toProtoMessage(direction)) - .setBaseCurrency(baseCurrency) - .setCounterCurrency(counterCurrency) - .setPaymentMethodId(offerPaymentMethod) - .setOfferDate(offerDate) - .setOfferUseMarketBasedPrice(offerUseMarketBasedPrice) - .setOfferMarketPriceMargin(offerMarketPriceMargin) - .setOfferAmount(offerAmount) - .setOfferMinAmount(offerMinAmount) - .setOfferId(offerId) - .setTradePrice(tradePrice) - .setTradeAmount(tradeAmount) - .setTradeDate(tradeDate) - .setDepositTxId(depositTxId) - .setSignaturePubKeyBytes(ByteString.copyFrom(signaturePubKeyBytes)); - Optional.ofNullable(extraDataMap).ifPresent(builder::putAllExtraData); - return protobuf.StoragePayload.newBuilder().setTradeStatistics(builder).build(); - } - - public protobuf.TradeStatistics toProtoTradeStatistics() { - return toProtoMessage().getTradeStatistics(); - } - - public static TradeStatistics fromProto(protobuf.TradeStatistics proto) { - return new TradeStatistics( - OfferPayload.Direction.fromProto(proto.getDirection()), - proto.getBaseCurrency(), - proto.getCounterCurrency(), - proto.getPaymentMethodId(), - proto.getOfferDate(), - proto.getOfferUseMarketBasedPrice(), - proto.getOfferMarketPriceMargin(), - proto.getOfferAmount(), - proto.getOfferMinAmount(), - proto.getOfferId(), - proto.getTradePrice(), - proto.getTradeAmount(), - proto.getTradeDate(), - proto.getDepositTxId(), - proto.getSignaturePubKeyBytes().toByteArray(), - CollectionUtils.isEmpty(proto.getExtraDataMap()) ? null : proto.getExtraDataMap()); - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // Getters - /////////////////////////////////////////////////////////////////////////////////////////// - - @Override - public long getTTL() { - return TimeUnit.DAYS.toMillis(30); - } - - @Override - public PublicKey getOwnerPubKey() { - return signaturePubKey; - } - - public Date getTradeDate() { - return new Date(tradeDate); - } - - public Price getTradePrice() { - return Price.valueOf(getCurrencyCode(), tradePrice); - } - - public String getCurrencyCode() { - return baseCurrency.equals("BTC") ? counterCurrency : baseCurrency; - } - - public Coin getTradeAmount() { - return Coin.valueOf(tradeAmount); - } - - public Volume getTradeVolume() { - if (getTradePrice().getMonetary() instanceof Altcoin) - return new Volume(new AltcoinExchangeRate((Altcoin) getTradePrice().getMonetary()).coinToAltcoin(getTradeAmount())); - else - return new Volume(new ExchangeRate((Fiat) getTradePrice().getMonetary()).coinToFiat(getTradeAmount())); - } -} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java index 17a878c270c..7beee9864dd 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java @@ -61,7 +61,7 @@ /** * Serialized size is about 180-210 byte. Nov 2017 we have 5500 objects */ - +@Deprecated @Slf4j @Value public final class TradeStatistics2 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java index 52822e2f2e0..27024943903 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java @@ -29,6 +29,7 @@ import java.io.File; +import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -65,11 +66,29 @@ public String getFileName() { @Override public Map getMap() { - return store.getMap(); + // As it is used for data request and response and we do not want to send any old trade stat data anymore. + return new HashMap<>(); + } + + // We overwrite that method to receive old trade stats from the network. As we deactivated getMap to not deliver + // hashes we needed to use the getMapOfAllData method to actually store the data. + // That's a bit of a hack but it's just for transition and can be removed after a few months anyway. + // Alternatively we could create a new interface to handle it differently on the other client classes but that + // seems to be not justified as it is needed only temporarily. + @Override + protected PersistableNetworkPayload putIfAbsent(P2PDataStorage.ByteArray hash, PersistableNetworkPayload payload) { + PersistableNetworkPayload previous = getMapOfAllData().putIfAbsent(hash, payload); + return previous; + } + + @Override + protected void readFromResources(String postFix) { + // We do not attempt to read from resources as that file is not provided anymore + readStore(); } public Map getMapOfAllData() { - return getMap(); + return store.getMap(); } @Override diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java new file mode 100644 index 00000000000..e585eff1fa1 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java @@ -0,0 +1,355 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.core.monetary.Altcoin; +import bisq.core.monetary.AltcoinExchangeRate; +import bisq.core.monetary.Price; +import bisq.core.monetary.Volume; +import bisq.core.offer.OfferUtil; + +import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; +import bisq.common.crypto.Hash; +import bisq.common.proto.ProtoUtil; +import bisq.common.util.CollectionUtils; +import bisq.common.util.ExtraDataMapValidator; +import bisq.common.util.JsonExclude; +import bisq.common.util.Utilities; + +import com.google.protobuf.ByteString; + +import org.bitcoinj.core.Coin; +import org.bitcoinj.utils.ExchangeRate; +import org.bitcoinj.utils.Fiat; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Charsets; + +import java.util.Arrays; +import java.util.Date; +import java.util.Map; +import java.util.Optional; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nullable; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * This new trade statistics class uses only the bare minimum of data. + * Data size is about 50 bytes in average + */ +@Slf4j +@Getter +public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, + CapabilityRequiringPayload { + + // This enum must not change the order as we use the ordinal for storage to reduce data size. + // The payment method string can be quite long and would consume 15% more space. + // When we get a new payment method we can add it to the enum at the end. Old users would add it as string if not + // recognized. + private enum PaymentMethodMapper { + OK_PAY, + CASH_APP, + VENMO, + AUSTRALIA_PAYID, // seems there is a dev trade + UPHOLD, + MONEY_BEAM, + POPMONEY, + REVOLUT, + PERFECT_MONEY, + SEPA, + SEPA_INSTANT, + FASTER_PAYMENTS, + NATIONAL_BANK, + JAPAN_BANK, + SAME_BANK, + SPECIFIC_BANKS, + SWISH, + ALI_PAY, + WECHAT_PAY, + CLEAR_X_CHANGE, + CHASE_QUICK_PAY, + INTERAC_E_TRANSFER, + US_POSTAL_MONEY_ORDER, + CASH_DEPOSIT, + MONEY_GRAM, + WESTERN_UNION, + HAL_CASH, + F2F, + BLOCK_CHAINS, + PROMPT_PAY, + ADVANCED_CASH, + BLOCK_CHAINS_INSTANT + } + + private final String currency; + private final long price; + private final long amount; + private final String paymentMethod; + // As only seller is publishing it is the sellers trade date + private final long date; + + // Old converted trade stat objects might not have it set + @Nullable + @JsonExclude + private String mediator; + @Nullable + @JsonExclude + private String refundAgent; + + // todo should we add referrerId as well? get added to extra map atm but not used so far + + // Hash get set in constructor from json of all the other data fields (with hash = null). + @JsonExclude + private final byte[] hash; + // Should be only used in emergency case if we need to add data but do not want to break backward compatibility + // at the P2P network storage checks. The hash of the object will be used to verify if the data is valid. Any new + // field in a class would break that hash and therefore break the storage mechanism. + @Nullable + @JsonExclude + private final Map extraDataMap; + + public TradeStatistics3(String currency, + long price, + long amount, + String paymentMethod, + long date, + String mediator, + String refundAgent, + @Nullable Map extraDataMap) { + this(currency, + price, + amount, + paymentMethod, + date, + mediator, + refundAgent, + extraDataMap, + null); + } + + // Used from conversion method where we use the hash of the TradeStatistics2 objects to avoid duplicate entries + public TradeStatistics3(String currency, + long price, + long amount, + String paymentMethod, + long date, + String mediator, + String refundAgent, + @Nullable byte[] hash) { + this(currency, + price, + amount, + paymentMethod, + date, + mediator, + refundAgent, + null, + hash); + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // PROTO BUFFER + /////////////////////////////////////////////////////////////////////////////////////////// + + @VisibleForTesting + public TradeStatistics3(String currency, + long price, + long amount, + String paymentMethod, + long date, + @Nullable String mediator, + @Nullable String refundAgent, + @Nullable Map extraDataMap, + @Nullable byte[] hash) { + this.currency = currency; + this.price = price; + this.amount = amount; + String tempPaymentMethod; + try { + tempPaymentMethod = String.valueOf(PaymentMethodMapper.valueOf(paymentMethod).ordinal()); + } catch (Throwable t) { + tempPaymentMethod = paymentMethod; + } + this.paymentMethod = tempPaymentMethod; + this.date = date; + this.mediator = mediator; + this.refundAgent = refundAgent; + this.extraDataMap = ExtraDataMapValidator.getValidatedExtraDataMap(extraDataMap); + + this.hash = hash == null ? createHash() : hash; + } + + public byte[] createHash() { + // We create hash from all fields excluding hash itself. We use json as simple data serialisation. + // TradeDate is different for both peers so we ignore it for hash. ExtraDataMap is ignored as well as at + // software updates we might have different entries which would cause a different hash. + return Hash.getSha256Ripemd160hash(Utilities.objectToJson(this).getBytes(Charsets.UTF_8)); + } + + private protobuf.TradeStatistics3.Builder getBuilder() { + protobuf.TradeStatistics3.Builder builder = protobuf.TradeStatistics3.newBuilder() + .setCurrency(currency) + .setPrice(price) + .setAmount(amount) + .setPaymentMethod(paymentMethod) + .setDate(date) + .setHash(ByteString.copyFrom(hash)); + Optional.ofNullable(mediator).ifPresent(builder::setMediator); + Optional.ofNullable(refundAgent).ifPresent(builder::setRefundAgent); + Optional.ofNullable(extraDataMap).ifPresent(builder::putAllExtraData); + return builder; + } + + public protobuf.TradeStatistics3 toProtoTradeStatistics3() { + return getBuilder().build(); + } + + @Override + public protobuf.PersistableNetworkPayload toProtoMessage() { + return protobuf.PersistableNetworkPayload.newBuilder().setTradeStatistics3(getBuilder()).build(); + } + + public static TradeStatistics3 fromProto(protobuf.TradeStatistics3 proto) { + return new TradeStatistics3( + proto.getCurrency(), + proto.getPrice(), + proto.getAmount(), + proto.getPaymentMethod(), + proto.getDate(), + ProtoUtil.stringOrNullFromProto(proto.getMediator()), + ProtoUtil.stringOrNullFromProto(proto.getRefundAgent()), + CollectionUtils.isEmpty(proto.getExtraDataMap()) ? null : proto.getExtraDataMap(), + proto.getHash().toByteArray()); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // API + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public byte[] getHash() { + return hash; + } + + @Override + public boolean verifyHashSize() { + checkNotNull(hash, "hash must not be null"); + return hash.length == 20; + } + + @Override + public Capabilities getRequiredCapabilities() { + return new Capabilities(Capability.TRADE_STATISTICS_3); + } + + public void pruneOptionalData() { + mediator = null; + refundAgent = null; + } + + public String getPaymentMethod() { + try { + return PaymentMethodMapper.values()[Integer.parseInt(paymentMethod)].name(); + } catch (Throwable ignore) { + return paymentMethod; + } + } + + public Date getTradeDate() { + return new Date(date); + } + + public Price getTradePrice() { + return Price.valueOf(currency, price); + } + + public Coin getTradeAmount() { + return Coin.valueOf(amount); + } + + public Volume getTradeVolume() { + if (getTradePrice().getMonetary() instanceof Altcoin) { + return new Volume(new AltcoinExchangeRate((Altcoin) getTradePrice().getMonetary()).coinToAltcoin(getTradeAmount())); + } else { + Volume volume = new Volume(new ExchangeRate((Fiat) getTradePrice().getMonetary()).coinToFiat(getTradeAmount())); + return OfferUtil.getRoundedFiatVolume(volume); + } + } + + public boolean isValid() { + return amount > 0 && + price > 0 && + date > 0 && + paymentMethod != null && + !paymentMethod.isEmpty() && + currency != null && + !currency.isEmpty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TradeStatistics3)) return false; + + TradeStatistics3 that = (TradeStatistics3) o; + + if (price != that.price) return false; + if (amount != that.amount) return false; + if (date != that.date) return false; + if (currency != null ? !currency.equals(that.currency) : that.currency != null) return false; + if (paymentMethod != null ? !paymentMethod.equals(that.paymentMethod) : that.paymentMethod != null) + return false; + return Arrays.equals(hash, that.hash); + } + + @Override + public int hashCode() { + int result = currency != null ? currency.hashCode() : 0; + result = 31 * result + (int) (price ^ (price >>> 32)); + result = 31 * result + (int) (amount ^ (amount >>> 32)); + result = 31 * result + (paymentMethod != null ? paymentMethod.hashCode() : 0); + result = 31 * result + (int) (date ^ (date >>> 32)); + result = 31 * result + Arrays.hashCode(hash); + return result; + } + + @Override + public String toString() { + return "TradeStatistics3{" + + "\n currency='" + currency + '\'' + + ",\n price=" + price + + ",\n amount=" + amount + + ",\n paymentMethod='" + paymentMethod + '\'' + + ",\n date=" + date + + ",\n mediator='" + mediator + '\'' + + ",\n refundAgent='" + refundAgent + '\'' + + ",\n hash=" + Utilities.bytesAsHexString(hash) + + ",\n extraDataMap=" + extraDataMap + + "\n}"; + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java new file mode 100644 index 00000000000..16398412319 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java @@ -0,0 +1,84 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.persistence.HistoricalDataStoreService; + +import bisq.common.config.Config; +import bisq.common.persistence.PersistenceManager; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; + +import java.io.File; + +import lombok.extern.slf4j.Slf4j; + +@Singleton +@Slf4j +public class TradeStatistics3StorageService extends HistoricalDataStoreService { + private static final String FILE_NAME = "TradeStatistics3Store"; + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Constructor + /////////////////////////////////////////////////////////////////////////////////////////// + + @Inject + public TradeStatistics3StorageService(@Named(Config.STORAGE_DIR) File storageDir, + PersistenceManager persistenceManager) { + super(storageDir, persistenceManager); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // API + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public String getFileName() { + return FILE_NAME; + } + + @Override + protected void initializePersistenceManager() { + persistenceManager.initialize(store, PersistenceManager.Source.NETWORK); + } + + @Override + public boolean canHandle(PersistableNetworkPayload payload) { + return payload instanceof TradeStatistics3; + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Protected + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + protected TradeStatistics3Store createStore() { + return new TradeStatistics3Store(); + } + + public void persistNow() { + persistenceManager.persistNow(() -> { + }); + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java new file mode 100644 index 00000000000..7c79778d4e1 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java @@ -0,0 +1,73 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.network.p2p.storage.P2PDataStorage; +import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore; + +import com.google.protobuf.Message; + +import java.util.List; +import java.util.stream.Collectors; + +import lombok.extern.slf4j.Slf4j; + +/** + * We store only the payload in the PB file to save disc space. The hash of the payload can be created anyway and + * is only used as key in the map. So we have a hybrid data structure which is represented as list in the protobuffer + * definition and provide a hashMap for the domain access. + */ +@Slf4j +public class TradeStatistics3Store extends PersistableNetworkPayloadStore { + + TradeStatistics3Store() { + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // PROTO BUFFER + /////////////////////////////////////////////////////////////////////////////////////////// + + private TradeStatistics3Store(List list) { + list.forEach(item -> map.put(new P2PDataStorage.ByteArray(item.getHash()), item)); + } + + public Message toProtoMessage() { + return protobuf.PersistableEnvelope.newBuilder() + .setTradeStatistics3Store(getBuilder()) + .build(); + } + + private protobuf.TradeStatistics3Store.Builder getBuilder() { + List protoList = map.values().stream() + .map(payload -> (TradeStatistics3) payload) + .map(TradeStatistics3::toProtoTradeStatistics3) + .collect(Collectors.toList()); + return protobuf.TradeStatistics3Store.newBuilder().addAllItems(protoList); + } + + public static TradeStatistics3Store fromProto(protobuf.TradeStatistics3Store proto) { + List list = proto.getItemsList().stream() + .map(TradeStatistics3::fromProto).collect(Collectors.toList()); + return new TradeStatistics3Store(list); + } + + public boolean containsKey(P2PDataStorage.ByteArray hash) { + return map.containsKey(hash); + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java new file mode 100644 index 00000000000..137c7d9ee85 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java @@ -0,0 +1,182 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.core.offer.availability.DisputeAgentSelection; + +import bisq.network.p2p.BootstrapListener; +import bisq.network.p2p.P2PService; +import bisq.network.p2p.storage.P2PDataStorage; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService; + +import bisq.common.UserThread; +import bisq.common.config.Config; +import bisq.common.file.FileUtil; +import bisq.common.util.Utilities; + +import com.google.inject.Inject; + +import javax.inject.Named; +import javax.inject.Singleton; + +import java.io.File; +import java.io.IOException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; + +import lombok.extern.slf4j.Slf4j; + +@Singleton +@Slf4j +public class TradeStatisticsConverter { + + private ExecutorService executor; + + @Inject + public TradeStatisticsConverter(P2PService p2PService, + P2PDataStorage p2PDataStorage, + TradeStatistics2StorageService tradeStatistics2StorageService, + TradeStatistics3StorageService tradeStatistics3StorageService, + AppendOnlyDataStoreService appendOnlyDataStoreService, + @Named(Config.STORAGE_DIR) File storageDir) { + File tradeStatistics2Store = new File(storageDir, "TradeStatistics2Store"); + appendOnlyDataStoreService.addService(tradeStatistics2StorageService); + + p2PService.addP2PServiceListener(new BootstrapListener() { + + @Override + public void onTorNodeReady() { + if (!tradeStatistics2Store.exists()) { + return; + } + executor = Utilities.getSingleThreadExecutor("TradeStatisticsConverter"); + executor.submit(() -> { + // We convert early once tor is initialized but still not ready to receive data + Map tempMap = new HashMap<>(); + convertToTradeStatistics3(tradeStatistics2StorageService.getMapOfAllData().values()) + .forEach(e -> tempMap.put(new P2PDataStorage.ByteArray(e.getHash()), e)); + + // We map to user thread to avoid potential threading issues + UserThread.execute(() -> { + tradeStatistics3StorageService.getMapOfLiveData().putAll(tempMap); + tradeStatistics3StorageService.persistNow(); + }); + + try { + log.info("We delete now the old trade statistics file as it was converted to the new format."); + FileUtil.deleteFileIfExists(tradeStatistics2Store); + } catch (IOException e) { + e.printStackTrace(); + log.error(e.toString()); + } + }); + } + + @Override + public void onUpdatedDataReceived() { + } + }); + + // We listen to old TradeStatistics2 objects, convert and store them and rebroadcast. + p2PDataStorage.addAppendOnlyDataStoreListener(payload -> { + if (payload instanceof TradeStatistics2) { + TradeStatistics3 tradeStatistics3 = convertToTradeStatistics3((TradeStatistics2) payload, true); + // We add it to the p2PDataStorage, which handles to get the data stored in the maps and maybe + // re-broadcast as tradeStatistics3 object if not already received. + p2PDataStorage.addPersistableNetworkPayload(tradeStatistics3, null, true); + } + }); + } + + public void shutDown() { + if (executor != null) + executor.shutdown(); + } + + private static List convertToTradeStatistics3(Collection persistableNetworkPayloads) { + List list = new ArrayList<>(); + long ts = System.currentTimeMillis(); + + // We might have duplicate entries from both traders as the trade date was different from old clients. + // This should not be the case with converting old persisted data as we did filter those out but it is the case + // when we receive old trade stat objects from the network of 2 not updated traders. + // The hash was ignoring the trade date so we use that to get a unique list + Map mapWithoutDuplicates = new HashMap<>(); + persistableNetworkPayloads.stream() + .filter(e -> e instanceof TradeStatistics2) + .map(e -> (TradeStatistics2) e) + .filter(TradeStatistics2::isValid) + .forEach(e -> mapWithoutDuplicates.putIfAbsent(new P2PDataStorage.ByteArray(e.getHash()), e)); + + log.info("We convert the existing {} trade statistics objects to the new format.", mapWithoutDuplicates.size()); + + mapWithoutDuplicates.values().stream() + .map(e -> convertToTradeStatistics3(e, false)) + .filter(TradeStatistics3::isValid) + .forEach(list::add); + + int size = list.size(); + log.info("Conversion to {} new trade statistic objects has been completed after {} ms", + size, System.currentTimeMillis() - ts); + + // We prune mediator and refundAgent data from all objects but the last 100 as we only use the + // last 100 entries (DisputeAgentSelection.LOOK_BACK_RANGE). + list.sort(Comparator.comparing(TradeStatistics3::getDate)); + if (size > DisputeAgentSelection.LOOK_BACK_RANGE) { + int start = size - DisputeAgentSelection.LOOK_BACK_RANGE; + for (int i = start; i < size; i++) { + TradeStatistics3 tradeStatistics3 = list.get(i); + tradeStatistics3.pruneOptionalData(); + } + } + + return list; + } + + private static TradeStatistics3 convertToTradeStatistics3(TradeStatistics2 tradeStatistics2, boolean fromNetwork) { + Map extraDataMap = tradeStatistics2.getExtraDataMap(); + String mediator = extraDataMap != null ? extraDataMap.get(TradeStatistics2.MEDIATOR_ADDRESS) : null; + String refundAgent = extraDataMap != null ? extraDataMap.get(TradeStatistics2.REFUND_AGENT_ADDRESS) : null; + long time = tradeStatistics2.getTradeDate().getTime(); + byte[] hash = null; + if (fromNetwork) { + // We need to avoid that we duplicate tradeStatistics2 objects in case both traders have not udpated yet. + // Before v1.4.0 both traders published the trade statistics. If one trader has updated he will check + // the capabilities of the peer and if the peer has not updated he will leave publishing to the peer, so we + // do not have the problem of duplicated objects. + // To ensure we add only one object we will use the hash of the tradeStatistics2 object which is the same + // for both traders as it excluded the trade date which is different for both. + hash = tradeStatistics2.getHash(); + } + return new TradeStatistics3(tradeStatistics2.getCurrencyCode(), + tradeStatistics2.getTradePrice().getValue(), + tradeStatistics2.getTradeAmount().getValue(), + tradeStatistics2.getOfferPaymentMethod(), + time, + mediator, + refundAgent, + hash); + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java index fbf675dd8e5..ebff70616be 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java @@ -21,7 +21,6 @@ import bisq.core.locale.Res; import bisq.core.monetary.Price; import bisq.core.monetary.Volume; -import bisq.core.offer.OfferPayload; import bisq.common.util.MathUtils; @@ -38,65 +37,44 @@ @ToString @Slf4j public final class TradeStatisticsForJson { - public final String currency; - public final OfferPayload.Direction direction; public final long tradePrice; public final long tradeAmount; public final long tradeDate; public final String paymentMethod; - public final long offerDate; - public final boolean useMarketBasedPrice; - public final double marketPriceMargin; - public final long offerAmount; - public final long offerMinAmount; - public final String offerId; - public final String depositTxId; // primaryMarket fields are based on industry standard where primaryMarket is always in the focus (in the app BTC is always in the focus - will be changed in a larger refactoring once) public String currencyPair; - public OfferPayload.Direction primaryMarketDirection; public long primaryMarketTradePrice; public long primaryMarketTradeAmount; public long primaryMarketTradeVolume; - public TradeStatisticsForJson(TradeStatistics2 tradeStatistics) { - this.direction = OfferPayload.Direction.valueOf(tradeStatistics.getDirection().name()); - this.currency = tradeStatistics.getCurrencyCode(); - this.paymentMethod = tradeStatistics.getOfferPaymentMethod(); - this.offerDate = tradeStatistics.getOfferDate(); - this.useMarketBasedPrice = tradeStatistics.isOfferUseMarketBasedPrice(); - this.marketPriceMargin = tradeStatistics.getOfferMarketPriceMargin(); - this.offerAmount = tradeStatistics.getOfferAmount(); - this.offerMinAmount = tradeStatistics.getOfferMinAmount(); - this.offerId = tradeStatistics.getOfferId(); - this.tradePrice = tradeStatistics.getTradePrice().getValue(); - this.tradeAmount = tradeStatistics.getTradeAmount().getValue(); - this.tradeDate = tradeStatistics.getTradeDate().getTime(); - this.depositTxId = tradeStatistics.getDepositTxId(); + public TradeStatisticsForJson(TradeStatistics3 tradeStatistics) { + this.currency = tradeStatistics.getCurrency(); + this.paymentMethod = tradeStatistics.getPaymentMethod(); + this.tradePrice = tradeStatistics.getPrice(); + this.tradeAmount = tradeStatistics.getAmount(); + this.tradeDate = tradeStatistics.getDate(); try { - final Price tradePrice = getTradePrice(); + Price tradePrice = getTradePrice(); if (CurrencyUtil.isCryptoCurrency(currency)) { - primaryMarketDirection = direction == OfferPayload.Direction.BUY ? OfferPayload.Direction.SELL : OfferPayload.Direction.BUY; currencyPair = currency + "/" + Res.getBaseCurrencyCode(); - primaryMarketTradePrice = tradePrice.getValue(); - - primaryMarketTradeAmount = getTradeVolume() != null ? getTradeVolume().getValue() : 0; + primaryMarketTradeAmount = getTradeVolume() != null ? + getTradeVolume().getValue() : + 0; primaryMarketTradeVolume = getTradeAmount().getValue(); } else { - primaryMarketDirection = direction; currencyPair = Res.getBaseCurrencyCode() + "/" + currency; - // we use precision 4 for fiat based price but on the markets api we use precision 8 so we scale up by 10000 primaryMarketTradePrice = (long) MathUtils.scaleUpByPowerOf10(tradePrice.getValue(), 4); - primaryMarketTradeAmount = getTradeAmount().getValue(); // we use precision 4 for fiat but on the markets api we use precision 8 so we scale up by 10000 primaryMarketTradeVolume = getTradeVolume() != null ? - (long) MathUtils.scaleUpByPowerOf10(getTradeVolume().getValue(), 4) : 0; + (long) MathUtils.scaleUpByPowerOf10(getTradeVolume().getValue(), 4) : + 0; } } catch (Throwable t) { log.error(t.getMessage()); @@ -113,6 +91,10 @@ public Coin getTradeAmount() { } public Volume getTradeVolume() { - return getTradePrice().getVolumeByAmount(getTradeAmount()); + try { + return getTradePrice().getVolumeByAmount(getTradeAmount()); + } catch (Throwable t) { + return Volume.parse("0", currency); + } } } diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java index 4dbe86c45bb..dedc250f85a 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java @@ -32,6 +32,7 @@ import com.google.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import javafx.collections.FXCollections; import javafx.collections.ObservableSet; @@ -40,94 +41,85 @@ import java.util.ArrayList; import java.util.List; -import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +@Singleton @Slf4j public class TradeStatisticsManager { - - private final JsonFileManager jsonFileManager; private final P2PService p2PService; private final PriceFeedService priceFeedService; - private final TradeStatistics2StorageService tradeStatistics2StorageService; + private final TradeStatistics3StorageService tradeStatistics3StorageService; + private final TradeStatisticsConverter tradeStatisticsConverter; + private final File storageDir; private final boolean dumpStatistics; - private final ObservableSet observableTradeStatisticsSet = FXCollections.observableSet(); + private final ObservableSet observableTradeStatisticsSet = FXCollections.observableSet(); + private JsonFileManager jsonFileManager; @Inject public TradeStatisticsManager(P2PService p2PService, PriceFeedService priceFeedService, - TradeStatistics2StorageService tradeStatistics2StorageService, + TradeStatistics3StorageService tradeStatistics3StorageService, AppendOnlyDataStoreService appendOnlyDataStoreService, + TradeStatisticsConverter tradeStatisticsConverter, @Named(Config.STORAGE_DIR) File storageDir, @Named(Config.DUMP_STATISTICS) boolean dumpStatistics) { this.p2PService = p2PService; this.priceFeedService = priceFeedService; - this.tradeStatistics2StorageService = tradeStatistics2StorageService; + this.tradeStatistics3StorageService = tradeStatistics3StorageService; + this.tradeStatisticsConverter = tradeStatisticsConverter; + this.storageDir = storageDir; this.dumpStatistics = dumpStatistics; - jsonFileManager = new JsonFileManager(storageDir); - appendOnlyDataStoreService.addService(tradeStatistics2StorageService); + + appendOnlyDataStoreService.addService(tradeStatistics3StorageService); + } + + public void shutDown() { + tradeStatisticsConverter.shutDown(); + if (jsonFileManager != null) { + jsonFileManager.shutDown(); + } } public void onAllServicesInitialized() { p2PService.getP2PDataStorage().addAppendOnlyDataStoreListener(payload -> { - if (payload instanceof TradeStatistics2) - addToSet((TradeStatistics2) payload); + if (payload instanceof TradeStatistics3) { + TradeStatistics3 tradeStatistics = (TradeStatistics3) payload; + if (!tradeStatistics.isValid()) { + return; + } + observableTradeStatisticsSet.add(tradeStatistics); + priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); + maybeDumpStatistics(); + } }); - Set set = tradeStatistics2StorageService.getMapOfAllData().values().stream() - .filter(e -> e instanceof TradeStatistics2) - .map(e -> (TradeStatistics2) e) - .map(WrapperTradeStatistics2::new) - .distinct() - .map(WrapperTradeStatistics2::unwrap) - .filter(TradeStatistics2::isValid) + Set set = tradeStatistics3StorageService.getMapOfAllData().values().stream() + .filter(e -> e instanceof TradeStatistics3) + .map(e -> (TradeStatistics3) e) + .filter(TradeStatistics3::isValid) .collect(Collectors.toSet()); observableTradeStatisticsSet.addAll(set); - priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); - - dump(); + maybeDumpStatistics(); } - public ObservableSet getObservableTradeStatisticsSet() { + public ObservableSet getObservableTradeStatisticsSet() { return observableTradeStatisticsSet; } - private void addToSet(TradeStatistics2 tradeStatistics) { - if (!observableTradeStatisticsSet.contains(tradeStatistics)) { - Optional duplicate = observableTradeStatisticsSet.stream().filter( - e -> e.getOfferId().equals(tradeStatistics.getOfferId())).findAny(); - - if (duplicate.isPresent()) { - // TODO: Can be removed as soon as everyone uses v1.2.6+ - // Removes an existing object with a trade id if the new one matches the existing except - // for the deposit tx id - if (tradeStatistics.getDepositTxId() == null && - tradeStatistics.isValid() && - duplicate.get().compareTo(tradeStatistics) == 0) { - observableTradeStatisticsSet.remove(duplicate.get()); - } else { - return; - } - } - - if (!tradeStatistics.isValid()) { - return; - } - - observableTradeStatisticsSet.add(tradeStatistics); - priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); - dump(); + private void maybeDumpStatistics() { + if (!dumpStatistics) { + return; } - } - private void dump() { - if (dumpStatistics) { + if (jsonFileManager == null) { + jsonFileManager = new JsonFileManager(storageDir); + + // We only dump once the currencies as they do not change during runtime ArrayList fiatCurrencyList = CurrencyUtil.getAllSortedFiatCurrencies().stream() .map(e -> new CurrencyTuple(e.getCode(), e.getName(), 8)) .collect(Collectors.toCollection(ArrayList::new)); @@ -138,44 +130,14 @@ private void dump() { .collect(Collectors.toCollection(ArrayList::new)); cryptoCurrencyList.add(0, new CurrencyTuple(Res.getBaseCurrencyCode(), Res.getBaseCurrencyName(), 8)); jsonFileManager.writeToDisc(Utilities.objectToJson(cryptoCurrencyList), "crypto_currency_list"); - - // We store the statistics as json so it is easy for further processing (e.g. for web based services) - // TODO This is just a quick solution for storing to one file. - // 1 statistic entry has 500 bytes as json. - // Need a more scalable solution later when we get more volume. - // The flag will only be activated by dedicated nodes, so it should not be too critical for the moment, but needs to - // get improved. Maybe a LevelDB like DB...? Could be impl. in a headless version only. - List list = observableTradeStatisticsSet.stream().map(TradeStatisticsForJson::new) - .sorted((o1, o2) -> (Long.compare(o2.tradeDate, o1.tradeDate))) - .collect(Collectors.toList()); - TradeStatisticsForJson[] array = new TradeStatisticsForJson[list.size()]; - list.toArray(array); - jsonFileManager.writeToDisc(Utilities.objectToJson(array), "trade_statistics"); - } - } - - static class WrapperTradeStatistics2 { - private final TradeStatistics2 tradeStatistics; - - public WrapperTradeStatistics2(TradeStatistics2 tradeStatistics) { - this.tradeStatistics = tradeStatistics; - } - - public TradeStatistics2 unwrap() { - return this.tradeStatistics; } - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - var wrapper = (WrapperTradeStatistics2) obj; - return Objects.equals(tradeStatistics.getOfferId(), wrapper.tradeStatistics.getOfferId()); - } - - @Override - public int hashCode() { - return Objects.hash(tradeStatistics.getOfferId()); - } + List list = observableTradeStatisticsSet.stream() + .map(TradeStatisticsForJson::new) + .sorted((o1, o2) -> (Long.compare(o2.tradeDate, o1.tradeDate))) + .collect(Collectors.toList()); + TradeStatisticsForJson[] array = new TradeStatisticsForJson[list.size()]; + list.toArray(array); + jsonFileManager.writeToDisc(Utilities.objectToJson(array), "trade_statistics"); } } diff --git a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Maker.java b/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Maker.java deleted file mode 100644 index 3c49ce2d5a6..00000000000 --- a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Maker.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import bisq.core.monetary.Price; -import bisq.core.offer.OfferPayload; - -import org.bitcoinj.core.Coin; - -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; - -import com.natpryce.makeiteasy.Instantiator; -import com.natpryce.makeiteasy.Maker; -import com.natpryce.makeiteasy.Property; - -import static com.natpryce.makeiteasy.MakeItEasy.a; - -public class TradeStatistics2Maker { - - public static final Property date = new Property<>(); - public static final Property depositTxId = new Property<>(); - public static final Property tradeAmount = new Property<>(); - - public static final Instantiator TradeStatistic2 = lookup -> { - Calendar calendar = Calendar.getInstance(); - calendar.set(2016, 3, 19); - - return new TradeStatistics2( - new OfferPayload("1234", - 0L, - null, - null, - OfferPayload.Direction.BUY, - 100000L, - 0.0, - false, - 100000L, - 100000L, - "BTC", - "USD", - null, - null, - "SEPA", - "", - null, - null, - null, - null, - null, - "", - 0L, - 0L, - 0L, - false, - 0L, - 0L, - 0L, - 0L, - false, - false, - 0L, - 0L, - false, - null, - null, - 0), - Price.valueOf("BTC", 100000L), - lookup.valueOf(tradeAmount, Coin.SATOSHI), - lookup.valueOf(date, new Date(calendar.getTimeInMillis())), - lookup.valueOf(depositTxId, "123456"), - Collections.emptyMap()); - }; - public static final Maker dayZeroTrade = a(TradeStatistic2); -} diff --git a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Test.java b/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Test.java deleted file mode 100644 index 725d18d5c7c..00000000000 --- a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Test.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import org.junit.Test; - -import static bisq.core.trade.statistics.TradeStatistics2Maker.dayZeroTrade; -import static bisq.core.trade.statistics.TradeStatistics2Maker.depositTxId; -import static com.natpryce.makeiteasy.MakeItEasy.make; -import static com.natpryce.makeiteasy.MakeItEasy.withNull; -import static org.junit.Assert.assertTrue; - - -public class TradeStatistics2Test { - - @Test - public void isValid_WithDepositTxId() { - - TradeStatistics2 tradeStatistic = make(dayZeroTrade); - - assertTrue(tradeStatistic.isValid()); - } - - @Test - public void isValid_WithEmptyDepositTxId() { - TradeStatistics2 tradeStatistic = make(dayZeroTrade.but(withNull(depositTxId))); - - assertTrue(tradeStatistic.isValid()); - } -} diff --git a/core/src/test/java/bisq/core/trade/statistics/TradeStatisticsManagerTest.java b/core/src/test/java/bisq/core/trade/statistics/TradeStatisticsManagerTest.java deleted file mode 100644 index 44b84d13848..00000000000 --- a/core/src/test/java/bisq/core/trade/statistics/TradeStatisticsManagerTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import bisq.core.provider.price.PriceFeedService; - -import bisq.network.p2p.P2PService; -import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreListener; -import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService; - -import org.bitcoinj.core.Coin; - -import java.io.File; - -import org.mockito.ArgumentCaptor; - -import org.junit.Before; -import org.junit.Test; - -import static bisq.core.trade.statistics.TradeStatistics2Maker.dayZeroTrade; -import static bisq.core.trade.statistics.TradeStatistics2Maker.depositTxId; -import static bisq.core.trade.statistics.TradeStatistics2Maker.tradeAmount; -import static com.natpryce.makeiteasy.MakeItEasy.make; -import static com.natpryce.makeiteasy.MakeItEasy.with; -import static com.natpryce.makeiteasy.MakeItEasy.withNull; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class TradeStatisticsManagerTest { - - private TradeStatisticsManager manager; - private TradeStatistics2 tradeWithNullDepositTxId; - private ArgumentCaptor listenerArgumentCaptor; - - @Before - public void prepareMocksAndObjects() { - P2PService p2PService = mock(P2PService.class); - P2PDataStorage p2PDataStorage = mock(P2PDataStorage.class); - File storageDir = mock(File.class); - TradeStatistics2StorageService tradeStatistics2StorageService = mock(TradeStatistics2StorageService.class); - PriceFeedService priceFeedService = mock(PriceFeedService.class); - - AppendOnlyDataStoreService appendOnlyDataStoreService = mock(AppendOnlyDataStoreService.class); - when(p2PService.getP2PDataStorage()).thenReturn(p2PDataStorage); - - manager = new TradeStatisticsManager(p2PService, priceFeedService, - tradeStatistics2StorageService, appendOnlyDataStoreService, storageDir, false); - - tradeWithNullDepositTxId = make(dayZeroTrade.but(withNull(depositTxId))); - - manager.onAllServicesInitialized(); - listenerArgumentCaptor = ArgumentCaptor.forClass(AppendOnlyDataStoreListener.class); - verify(p2PDataStorage).addAppendOnlyDataStoreListener(listenerArgumentCaptor.capture()); - - } - - @Test - public void addToSet_ObjectWithNullDepositTxId() { - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - } - - @Test - public void addToSet_RemoveExistingObjectIfObjectWithNullDepositTxIdIsAdded() { - TradeStatistics2 tradeWithDepositTxId = make(dayZeroTrade); - - listenerArgumentCaptor.getValue().onAdded(tradeWithDepositTxId); - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - - assertFalse(manager.getObservableTradeStatisticsSet().contains(tradeWithDepositTxId)); - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - } - - @Test - public void addToSet_NotRemoveExistingObjectIfObjectsNotEqual() { - TradeStatistics2 tradeWithDepositTxId = make(dayZeroTrade.but(with(tradeAmount, Coin.FIFTY_COINS))); - - listenerArgumentCaptor.getValue().onAdded(tradeWithDepositTxId); - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithDepositTxId)); - assertFalse(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - } - - @Test - public void addToSet_IgnoreObjectIfObjectWithNullDepositTxIdAlreadyExists() { - TradeStatistics2 tradeWithDepositTxId = make(dayZeroTrade); - - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - listenerArgumentCaptor.getValue().onAdded(tradeWithDepositTxId); - - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - assertFalse(manager.getObservableTradeStatisticsSet().contains(tradeWithDepositTxId)); - } -} diff --git a/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java b/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java index 281fc121185..4c98e939af3 100644 --- a/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java +++ b/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java @@ -1,7 +1,7 @@ package bisq.daemon.grpc; import bisq.core.api.CoreApi; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.proto.grpc.GetTradeStatisticsGrpc; import bisq.proto.grpc.GetTradeStatisticsReply; @@ -27,7 +27,7 @@ public void getTradeStatistics(GetTradeStatisticsRequest req, StreamObserver responseObserver) { var tradeStatistics = coreApi.getTradeStatistics().stream() - .map(TradeStatistics2::toProtoTradeStatistics2) + .map(TradeStatistics3::toProtoTradeStatistics3) .collect(Collectors.toList()); var reply = GetTradeStatisticsReply.newBuilder().addAllTradeStatistics(tradeStatistics).build(); diff --git a/desktop/src/main/java/bisq/desktop/components/AddressTextField.java b/desktop/src/main/java/bisq/desktop/components/AddressTextField.java index 9bc06ce5147..f9ca98a1130 100644 --- a/desktop/src/main/java/bisq/desktop/components/AddressTextField.java +++ b/desktop/src/main/java/bisq/desktop/components/AddressTextField.java @@ -78,9 +78,6 @@ public AddressTextField(String label) { }); textField.focusTraversableProperty().set(focusTraversableProperty().get()); - //TODO app wide focus - //focusedProperty().addListener((ov, oldValue, newValue) -> textField.requestFocus()); - Label extWalletIcon = new Label(); extWalletIcon.setLayoutY(3); extWalletIcon.getStyleClass().addAll("icon", "highlight"); diff --git a/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java b/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java index 97d61fd07bf..a74a4ae2e3b 100644 --- a/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java +++ b/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java @@ -74,9 +74,6 @@ public BsqAddressTextField() { }); textField.focusTraversableProperty().set(focusTraversableProperty().get()); - //TODO app wide focus - //focusedProperty().addListener((ov, oldValue, newValue) -> textField.requestFocus()); - Label copyIcon = new Label(); copyIcon.setLayoutY(3); diff --git a/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java b/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java index fe26aebeab7..7f2892379ca 100644 --- a/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java +++ b/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java @@ -79,9 +79,6 @@ public TextFieldWithCopyIcon(String customStyleClass) { AnchorPane.setRightAnchor(textField, 30.0); AnchorPane.setLeftAnchor(textField, 0.0); textField.focusTraversableProperty().set(focusTraversableProperty().get()); - //TODO app wide focus - //focusedProperty().addListener((ov, oldValue, newValue) -> textField.requestFocus()); - getChildren().addAll(textField, copyIcon); } diff --git a/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java b/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java index 4af167a5940..656d126f86d 100644 --- a/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java +++ b/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java @@ -64,8 +64,6 @@ import static bisq.desktop.util.FormBuilder.addTitledGroupBg; import static bisq.desktop.util.FormBuilder.addTopLabelTextField; -// TODO translation string keys should renamed to be more generic. -// Lets do it for 1.1.7 the translator have time to add new string. public abstract class AgentRegistrationView> extends ActivatableViewAndModel { diff --git a/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java b/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java index 24ef6601465..ed0257e0449 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java @@ -30,7 +30,7 @@ import bisq.core.monetary.Altcoin; import bisq.core.monetary.Price; import bisq.core.provider.price.PriceFeedService; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; import bisq.core.util.FormattingUtils; @@ -121,7 +121,7 @@ public class BsqDashboardView extends ActivatableView implements /////////////////////////////////////////////////////////////////////////////////////////// @Inject - private BsqDashboardView(DaoFacade daoFacade, + public BsqDashboardView(DaoFacade daoFacade, TradeStatisticsManager tradeStatisticsManager, PriceFeedService priceFeedService, Preferences preferences, @@ -173,7 +173,7 @@ private void createKPIs() { marketCapTextField = addTopLabelReadOnlyTextField(root, ++gridRow, Res.get("dao.factsAndFigures.dashboard.marketCap")).second; - availableAmountTextField = FormBuilder.addTopLabelReadOnlyTextField(root, gridRow, 1, + availableAmountTextField = addTopLabelReadOnlyTextField(root, gridRow, 1, Res.get("dao.factsAndFigures.dashboard.availableAmount")).second; } @@ -289,17 +289,17 @@ private void updateChartData() { private void updateBsqPriceData() { seriesBSQPrice.getData().clear(); - Map> bsqPriceByDate = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals("BSQ")) - .sorted(Comparator.comparing(TradeStatistics2::getTradeDate)) - .collect(Collectors.groupingBy(item -> new java.sql.Date(item.getTradeDate().getTime()).toLocalDate() + Map> bsqPriceByDate = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() + .filter(e -> e.getCurrency().equals("BSQ")) + .sorted(Comparator.comparing(TradeStatistics3::getDate)) + .collect(Collectors.groupingBy(item -> new java.sql.Date(item.getDate()).toLocalDate() .with(ADJUSTERS.get(DAY)))); List> updatedBSQPrice = bsqPriceByDate.keySet().stream() .map(e -> { ZonedDateTime zonedDateTime = e.atStartOfDay(ZoneId.systemDefault()); return new XYChart.Data(zonedDateTime.toInstant().getEpochSecond(), bsqPriceByDate.get(e).stream() - .map(TradeStatistics2::getTradePrice) + .map(TradeStatistics3::getTradePrice) .mapToDouble(Price::getValue) .average() .orElse(Double.NaN) @@ -370,12 +370,12 @@ private void updateAveragePriceFields(TextField field90, TextFieldWithIcon field private long updateAveragePriceField(TextField textField, int days, boolean isUSDField) { Date pastXDays = getPastDate(days); - List bsqTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals("BSQ")) + List bsqTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() + .filter(e -> e.getCurrency().equals("BSQ")) .filter(e -> e.getTradeDate().after(pastXDays)) .collect(Collectors.toList()); - List usdTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals("USD")) + List usdTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() + .filter(e -> e.getCurrency().equals("USD")) .filter(e -> e.getTradeDate().after(pastXDays)) .collect(Collectors.toList()); long average = isUSDField ? getUSDAverage(bsqTradePastXDays, usdTradePastXDays) : @@ -391,11 +391,11 @@ private long updateAveragePriceField(TextField textField, int days, boolean isUS return average; } - private long getBTCAverage(List bsqList) { + private long getBTCAverage(List bsqList) { long accumulatedVolume = 0; long accumulatedAmount = 0; - for (TradeStatistics2 item : bsqList) { + for (TradeStatistics3 item : bsqList) { accumulatedVolume += item.getTradeVolume().getValue(); accumulatedAmount += item.getTradeAmount().getValue(); // Amount of BTC traded } @@ -406,17 +406,17 @@ private long getBTCAverage(List bsqList) { return averagePrice; } - private long getUSDAverage(List bsqList, List usdList) { + private long getUSDAverage(List bsqList, List usdList) { // Use next USD/BTC print as price to calculate BSQ/USD rate // Store each trade as amount of USD and amount of BSQ traded List> usdBsqList = new ArrayList<>(bsqList.size()); - usdList.sort(Comparator.comparing(o -> o.getTradeDate().getTime())); + usdList.sort(Comparator.comparing(TradeStatistics3::getDate)); var usdBTCPrice = 10000d; // Default to 10000 USD per BTC if there is no USD feed at all - for (TradeStatistics2 item : bsqList) { + for (TradeStatistics3 item : bsqList) { // Find usdprice for trade item usdBTCPrice = usdList.stream() - .filter(usd -> usd.getTradeDate().getTime() > item.getTradeDate().getTime()) + .filter(usd -> usd.getDate() > item.getDate()) .map(usd -> MathUtils.scaleDownByPowerOf10((double) usd.getTradePrice().getValue(), Fiat.SMALLEST_UNIT_EXPONENT)) .findFirst() diff --git a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java index 86b0c4d0d05..89d0acbf090 100644 --- a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java +++ b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java @@ -27,7 +27,6 @@ import bisq.core.offer.placeoffer.tasks.CreateMakerFeeTx; import bisq.core.offer.placeoffer.tasks.ValidateOffer; import bisq.core.trade.protocol.tasks.ApplyFilter; -import bisq.core.trade.protocol.tasks.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.VerifyPeersAccountAgeWitness; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessDelayedPayoutTxSignatureRequest; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessDepositTxAndDelayedPayoutTxMessage; @@ -56,6 +55,7 @@ import bisq.core.trade.protocol.tasks.seller.SellerProcessCounterCurrencyTransferStartedMessage; import bisq.core.trade.protocol.tasks.seller.SellerProcessDelayedPayoutTxSignatureResponse; import bisq.core.trade.protocol.tasks.seller.SellerPublishesDepositTx; +import bisq.core.trade.protocol.tasks.seller.SellerPublishesTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerSendDelayedPayoutTxSignatureRequest; import bisq.core.trade.protocol.tasks.seller.SellerSendPayoutTxPublishedMessage; import bisq.core.trade.protocol.tasks.seller.SellerSendsDepositTxAndDelayedPayoutTxMessage; @@ -145,7 +145,7 @@ public void initialize() { SellerFinalizesDelayedPayoutTx.class, SellerSendsDepositTxAndDelayedPayoutTxMessage.class, SellerPublishesDepositTx.class, - PublishTradeStatistics.class, + SellerPublishesTradeStatistics.class, SellerProcessCounterCurrencyTransferStartedMessage.class, ApplyFilter.class, @@ -179,7 +179,6 @@ public void initialize() { BuyerProcessDepositTxAndDelayedPayoutTxMessage.class, BuyerVerifiesFinalDelayedPayoutTx.class, - PublishTradeStatistics.class, ApplyFilter.class, MakerVerifyTakerFeePayment.class, @@ -216,7 +215,6 @@ public void initialize() { BuyerProcessDepositTxAndDelayedPayoutTxMessage.class, BuyerVerifiesFinalDelayedPayoutTx.class, - PublishTradeStatistics.class, ApplyFilter.class, TakerVerifyMakerFeePayment.class, @@ -248,7 +246,7 @@ public void initialize() { SellerFinalizesDelayedPayoutTx.class, SellerSendsDepositTxAndDelayedPayoutTxMessage.class, SellerPublishesDepositTx.class, - PublishTradeStatistics.class, + SellerPublishesTradeStatistics.class, SellerProcessCounterCurrencyTransferStartedMessage.class, ApplyFilter.class, diff --git a/desktop/src/main/java/bisq/desktop/main/market/MarketView.java b/desktop/src/main/java/bisq/desktop/main/market/MarketView.java index 15641ac3dc1..9a3544e06d0 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/MarketView.java +++ b/desktop/src/main/java/bisq/desktop/main/market/MarketView.java @@ -35,7 +35,7 @@ import bisq.core.locale.CurrencyUtil; import bisq.core.locale.Res; import bisq.core.offer.OfferPayload; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.util.FormattingUtils; import bisq.core.util.coin.CoinFormatter; @@ -82,7 +82,10 @@ public class MarketView extends ActivatableView { @Inject - public MarketView(CachingViewLoader viewLoader, P2PService p2PService, OfferBook offerBook, @Named(FormattingUtils.BTC_FORMATTER_KEY) CoinFormatter formatter, + public MarketView(CachingViewLoader viewLoader, + P2PService p2PService, + OfferBook offerBook, + @Named(FormattingUtils.BTC_FORMATTER_KEY) CoinFormatter formatter, Navigation navigation) { this.viewLoader = viewLoader; this.p2PService = p2PService; @@ -179,20 +182,19 @@ private String getAllTradesWithReferralId() { // If both traders had set it the tradeStatistics is only delivered once. // If both traders used a different referral ID then we would get 2 objects. List list = p2PService.getP2PDataStorage().getAppendOnlyDataStoreMap().values().stream() - .filter(e -> e instanceof TradeStatistics2) - .map(e -> (TradeStatistics2) e) - .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap() != null) - .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap().get(OfferPayload.REFERRAL_ID) != null) - .map(trade -> { + .filter(e -> e instanceof TradeStatistics3) + .map(e -> (TradeStatistics3) e) + .filter(tradeStatistics3 -> tradeStatistics3.getExtraDataMap() != null) + .filter(tradeStatistics3 -> tradeStatistics3.getExtraDataMap().get(OfferPayload.REFERRAL_ID) != null) + .map(tradeStatistics3 -> { StringBuilder sb = new StringBuilder(); - sb.append("Trade ID: ").append(trade.getOfferId()).append("\n") - .append("Date: ").append(DisplayUtils.formatDateTime(trade.getTradeDate())).append("\n") - .append("Market: ").append(CurrencyUtil.getCurrencyPair(trade.getCurrencyCode())).append("\n") - .append("Price: ").append(FormattingUtils.formatPrice(trade.getTradePrice())).append("\n") - .append("Amount: ").append(formatter.formatCoin(trade.getTradeAmount())).append("\n") - .append("Volume: ").append(DisplayUtils.formatVolume(trade.getTradeVolume())).append("\n") - .append("Payment method: ").append(Res.get(trade.getOfferPaymentMethod())).append("\n") - .append("ReferralID: ").append(trade.getExtraDataMap().get(OfferPayload.REFERRAL_ID)); + sb.append("Date: ").append(DisplayUtils.formatDateTime(tradeStatistics3.getTradeDate())).append("\n") + .append("Market: ").append(CurrencyUtil.getCurrencyPair(tradeStatistics3.getCurrency())).append("\n") + .append("Price: ").append(FormattingUtils.formatPrice(tradeStatistics3.getTradePrice())).append("\n") + .append("Amount: ").append(formatter.formatCoin(tradeStatistics3.getTradeAmount())).append("\n") + .append("Volume: ").append(DisplayUtils.formatVolume(tradeStatistics3.getTradeVolume())).append("\n") + .append("Payment method: ").append(Res.get(tradeStatistics3.getPaymentMethod())).append("\n") + .append("ReferralID: ").append(tradeStatistics3.getExtraDataMap().get(OfferPayload.REFERRAL_ID)); return sb.toString(); }) .collect(Collectors.toList()); diff --git a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java index 02a2a402e51..539ab5ffad1 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java +++ b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java @@ -298,7 +298,7 @@ public TableCell call( public void updateItem(final SpreadItem item, boolean empty) { super.updateItem(item, empty); if (item != null && !empty) { - // TODO maybe show exra colums with item.priceSpread and use real amount diff + // TODO maybe show extra columns with item.priceSpread and use real amount diff // not % based if (item.priceSpread != null) setText(item.percentage); diff --git a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java index 97aecedb3cb..e5ee933ca0d 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java +++ b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java @@ -172,7 +172,7 @@ private void update(ObservableList offerBookListItems) { else spread = bestBuyOfferPrice.subtract(bestSellOfferPrice); - // TODO maybe show extra colums with spread and use real amount diff + // TODO maybe show extra columns with spread and use real amount diff // not % based. e.g. diff between best buy and sell offer (of small amounts its a smaller gain) if (spread != null && marketPrice != null && marketPrice.isPriceAvailable()) { diff --git a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java index f26d3c1ea05..4a70b96ff43 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java +++ b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java @@ -34,8 +34,7 @@ import bisq.core.locale.Res; import bisq.core.monetary.Price; import bisq.core.monetary.Volume; -import bisq.core.offer.OfferPayload; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.util.FormattingUtils; import bisq.core.util.coin.CoinFormatter; @@ -103,7 +102,7 @@ public class TradesChartsView extends ActivatableViewAndModel tableView; + private TableView tableView; private AutocompleteComboBox currencyComboBox; private VolumeChart volumeChart; private CandleStickChart priceChart; @@ -118,12 +117,12 @@ public class TradesChartsView extends ActivatableViewAndModel timeUnitChangeListener; private ToggleGroup toggleGroup; private final ListChangeListener> itemsChangeListener; - private SortedList sortedList; + private SortedList sortedList; private Label nrOfTradeStatisticsLabel; - private ListChangeListener tradeStatisticsByCurrencyListener; + private ListChangeListener tradeStatisticsByCurrencyListener; private ChangeListener selectedTabIndexListener; private SingleSelectionModel tabPaneSelectionModel; - private TableColumn priceColumn, volumeColumn, marketColumn; + private TableColumn priceColumn, volumeColumn, marketColumn; @SuppressWarnings("FieldCanBeLocal") private MonadicBinding currencySelectionBinding; private Subscription currencySelectionSubscriber; @@ -550,7 +549,7 @@ private void createTable() { VBox.setVgrow(tableView, Priority.ALWAYS); // date - TableColumn dateColumn = new AutoTooltipTableColumn<>(Res.get("shared.dateTime")) { + TableColumn dateColumn = new AutoTooltipTableColumn<>(Res.get("shared.dateTime")) { { setMinWidth(240); setMaxWidth(240); @@ -561,11 +560,11 @@ private void createTable() { dateColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(DisplayUtils.formatDateTime(item.getTradeDate())); @@ -575,7 +574,7 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { }; } }); - dateColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradeDate)); + dateColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradeDate)); tableView.getColumns().add(dateColumn); // market @@ -590,21 +589,21 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { marketColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) - setText(CurrencyUtil.getCurrencyPair(item.getCurrencyCode())); + setText(CurrencyUtil.getCurrencyPair(item.getCurrency())); else setText(""); } }; } }); - marketColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradeDate)); + marketColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradeDate)); tableView.getColumns().add(marketColumn); // price @@ -614,11 +613,11 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { priceColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(FormattingUtils.formatPrice(item.getTradePrice())); @@ -628,21 +627,21 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { }; } }); - priceColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradePrice)); + priceColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradePrice)); tableView.getColumns().add(priceColumn); // amount - TableColumn amountColumn = new AutoTooltipTableColumn<>(Res.get("shared.amountWithCur", Res.getBaseCurrencyCode())); + TableColumn amountColumn = new AutoTooltipTableColumn<>(Res.get("shared.amountWithCur", Res.getBaseCurrencyCode())); amountColumn.getStyleClass().add("number-column"); amountColumn.setCellValueFactory((tradeStatistics) -> new ReadOnlyObjectWrapper<>(tradeStatistics.getValue())); amountColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setGraphic(new ColoredDecimalPlacesWithZerosText(formatter.formatCoin(item.getTradeAmount(), @@ -653,7 +652,7 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { }; } }); - amountColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradeAmount)); + amountColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradeAmount)); tableView.getColumns().add(amountColumn); // volume @@ -663,11 +662,11 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { volumeColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(model.showAllTradeCurrenciesProperty.get() ? @@ -687,17 +686,17 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { tableView.getColumns().add(volumeColumn); // paymentMethod - TableColumn paymentMethodColumn = new AutoTooltipTableColumn<>(Res.get("shared.paymentMethod")); + TableColumn paymentMethodColumn = new AutoTooltipTableColumn<>(Res.get("shared.paymentMethod")); paymentMethodColumn.getStyleClass().add("number-column"); paymentMethodColumn.setCellValueFactory((tradeStatistics) -> new ReadOnlyObjectWrapper<>(tradeStatistics.getValue())); paymentMethodColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(getPaymentMethodLabel(item)); @@ -710,30 +709,6 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { paymentMethodColumn.setComparator(Comparator.comparing(this::getPaymentMethodLabel)); tableView.getColumns().add(paymentMethodColumn); - // direction - TableColumn directionColumn = new AutoTooltipTableColumn<>(Res.get("shared.offerType")); - directionColumn.getStyleClass().addAll("number-column", "last-column"); - directionColumn.setCellValueFactory((tradeStatistics) -> new ReadOnlyObjectWrapper<>(tradeStatistics.getValue())); - directionColumn.setCellFactory( - new Callback<>() { - @Override - public TableCell call( - TableColumn column) { - return new TableCell<>() { - @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { - super.updateItem(item, empty); - if (item != null) - setText(getDirectionLabel(item)); - else - setText(""); - } - }; - } - }); - directionColumn.setComparator(Comparator.comparing(this::getDirectionLabel)); - tableView.getColumns().add(directionColumn); - tableView.setColumnResizePolicy(TableView.CONSTRAINED_RESIZE_POLICY); Label placeholder = new AutoTooltipLabel(Res.get("table.placeholder.noData")); placeholder.setWrapText(true); @@ -743,13 +718,8 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { } @NotNull - private String getDirectionLabel(TradeStatistics2 item) { - return DisplayUtils.getDirectionWithCode(OfferPayload.Direction.valueOf(item.getDirection().name()), item.getCurrencyCode()); - } - - @NotNull - private String getPaymentMethodLabel(TradeStatistics2 item) { - return Res.get(item.getOfferPaymentMethod()); + private String getPaymentMethodLabel(TradeStatistics3 item) { + return Res.get(item.getPaymentMethod()); } private void layout() { diff --git a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java index f070b5946b3..3bf125ad897 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java +++ b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java @@ -34,7 +34,7 @@ import bisq.core.locale.TradeCurrency; import bisq.core.monetary.Altcoin; import bisq.core.provider.price.PriceFeedService; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; @@ -97,18 +97,18 @@ public enum TickUnit { private final TradeStatisticsManager tradeStatisticsManager; final Preferences preferences; - private PriceFeedService priceFeedService; - private Navigation navigation; + private final PriceFeedService priceFeedService; + private final Navigation navigation; - private final SetChangeListener setChangeListener; + private final SetChangeListener setChangeListener; final ObjectProperty selectedTradeCurrencyProperty = new SimpleObjectProperty<>(); final BooleanProperty showAllTradeCurrenciesProperty = new SimpleBooleanProperty(false); private final CurrencyList currencyListItems; private final CurrencyListItem showAllCurrencyListItem = new CurrencyListItem(new CryptoCurrency(GUIUtil.SHOW_ALL_FLAG, ""), -1); - final ObservableList tradeStatisticsByCurrency = FXCollections.observableArrayList(); + final ObservableList tradeStatisticsByCurrency = FXCollections.observableArrayList(); final ObservableList> priceItems = FXCollections.observableArrayList(); final ObservableList> volumeItems = FXCollections.observableArrayList(); - private Map>> itemsPerInterval; + private Map>> itemsPerInterval; TickUnit tickUnit; final int maxTicks = 90; @@ -119,7 +119,8 @@ public enum TickUnit { /////////////////////////////////////////////////////////////////////////////////////////// @Inject - TradesChartsViewModel(TradeStatisticsManager tradeStatisticsManager, Preferences preferences, PriceFeedService priceFeedService, Navigation navigation) { + TradesChartsViewModel(TradeStatisticsManager tradeStatisticsManager, Preferences preferences, + PriceFeedService priceFeedService, Navigation navigation) { this.tradeStatisticsManager = tradeStatisticsManager; this.preferences = preferences; this.priceFeedService = priceFeedService; @@ -145,7 +146,7 @@ public enum TickUnit { private void fillTradeCurrencies() { // Don't use a set as we need all entries List tradeCurrencyList = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .flatMap(e -> CurrencyUtil.getTradeCurrency(e.getCurrencyCode()).stream()) + .flatMap(e -> CurrencyUtil.getTradeCurrency(e.getCurrency()).stream()) .collect(Collectors.toList()); currencyListItems.updateWithCurrencies(tradeCurrencyList, showAllCurrencyListItem); @@ -241,15 +242,15 @@ private void syncPriceFeedCurrency() { private void updateChartData() { tradeStatisticsByCurrency.setAll(tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> showAllTradeCurrenciesProperty.get() || e.getCurrencyCode().equals(getCurrencyCode())) + .filter(e -> showAllTradeCurrenciesProperty.get() || e.getCurrency().equals(getCurrencyCode())) .collect(Collectors.toList())); // Generate date range and create sets for all ticks itemsPerInterval = new HashMap<>(); Date time = new Date(); for (long i = maxTicks + 1; i >= 0; --i) { - Set set = new HashSet<>(); - Pair> pair = new Pair<>((Date) time.clone(), set); + Set set = new HashSet<>(); + Pair> pair = new Pair<>((Date) time.clone(), set); itemsPerInterval.put(i, pair); time.setTime(time.getTime() - 1); time = roundToTick(time, tickUnit); @@ -258,7 +259,7 @@ private void updateChartData() { // Get all entries for the defined time interval tradeStatisticsByCurrency.forEach(e -> { for (long i = maxTicks; i > 0; --i) { - Pair> p = itemsPerInterval.get(i); + Pair> p = itemsPerInterval.get(i); if (e.getTradeDate().after(p.getKey())) { p.getValue().add(e); break; @@ -283,7 +284,7 @@ private void updateChartData() { } @VisibleForTesting - CandleData getCandleData(long tick, Set set) { + CandleData getCandleData(long tick, Set set) { long open = 0; long close = 0; long high = 0; @@ -293,7 +294,7 @@ CandleData getCandleData(long tick, Set set) { long numTrades = set.size(); List tradePrices = new ArrayList<>(set.size()); - for (TradeStatistics2 item : set) { + for (TradeStatistics3 item : set) { long tradePriceAsLong = item.getTradePrice().getValue(); // Previously a check was done which inverted the low and high for cryptocurrencies. low = (low != 0) ? Math.min(low, tradePriceAsLong) : tradePriceAsLong; @@ -305,8 +306,8 @@ CandleData getCandleData(long tick, Set set) { } Collections.sort(tradePrices); - List list = new ArrayList<>(set); - list.sort(Comparator.comparingLong(o -> o.getTradeDate().getTime())); + List list = new ArrayList<>(set); + list.sort(Comparator.comparingLong(TradeStatistics3::getDate)); if (list.size() > 0) { open = list.get(0).getTradePrice().getValue(); close = list.get(list.size() - 1).getTradePrice().getValue(); diff --git a/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java b/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java index 44371697167..62869cd54e7 100644 --- a/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java +++ b/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java @@ -43,7 +43,7 @@ import bisq.core.provider.fee.FeeService; import bisq.core.provider.price.PriceFeedService; import bisq.core.trade.handlers.TransactionResultHandler; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; import bisq.core.user.User; @@ -344,9 +344,9 @@ private void setSuggestedSecurityDeposit(PaymentAccount paymentAccount) { var blocksRange = Restrictions.getLockTime(paymentAccount.getPaymentMethod().isAsset()); var startDate = new Date(System.currentTimeMillis() - blocksRange * 10 * 60000); var sortedRangeData = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals(getTradeCurrency().getCode())) + .filter(e -> e.getCurrency().equals(getTradeCurrency().getCode())) .filter(e -> e.getTradeDate().compareTo(startDate) >= 0) - .sorted(Comparator.comparing(TradeStatistics2::getTradeDate)) + .sorted(Comparator.comparing(TradeStatistics3::getTradeDate)) .collect(Collectors.toList()); var movingAverage = new MathUtils.MovingAverage(10, 0.2); double[] extremes = {Double.MAX_VALUE, Double.MIN_VALUE}; diff --git a/desktop/src/main/java/bisq/desktop/util/FormBuilder.java b/desktop/src/main/java/bisq/desktop/util/FormBuilder.java index 46db98c0153..f8061e04f99 100644 --- a/desktop/src/main/java/bisq/desktop/util/FormBuilder.java +++ b/desktop/src/main/java/bisq/desktop/util/FormBuilder.java @@ -332,7 +332,7 @@ public static Tuple3 addTopLabelTextField(GridPane gridP final Tuple2 topLabelWithVBox = addTopLabelWithVBox(gridPane, rowIndex, title, textField, top); - // TOD not 100% sure if that is a good idea.... + // TODO not 100% sure if that is a good idea.... //topLabelWithVBox.first.getStyleClass().add("jfx-text-field-top-label"); return new Tuple3<>(topLabelWithVBox.first, textField, topLabelWithVBox.second); diff --git a/desktop/src/main/resources/logback.xml b/desktop/src/main/resources/logback.xml index 8812d310de9..6b05588a4fe 100644 --- a/desktop/src/main/resources/logback.xml +++ b/desktop/src/main/resources/logback.xml @@ -11,7 +11,6 @@ - + diff --git a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java index f3223f5572f..8d3a7d25660 100644 --- a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java +++ b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java @@ -23,14 +23,12 @@ import bisq.core.locale.FiatCurrency; import bisq.core.monetary.Price; import bisq.core.offer.OfferPayload; +import bisq.core.payment.payload.PaymentMethod; import bisq.core.provider.price.PriceFeedService; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; -import bisq.common.crypto.KeyRing; -import bisq.common.crypto.KeyStorage; - import org.bitcoinj.core.Coin; import org.bitcoinj.utils.Fiat; @@ -49,9 +47,6 @@ import java.util.HashSet; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -63,9 +58,7 @@ public class TradesChartsViewModelTest { TradesChartsViewModel model; TradeStatisticsManager tradeStatisticsManager; - private static final Logger log = LoggerFactory.getLogger(TradesChartsViewModelTest.class); DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - private KeyRing keyRing; private File dir; OfferPayload offer = new OfferPayload(null, 0, @@ -106,6 +99,7 @@ public class TradesChartsViewModelTest { null, 1 ); + @Before public void setup() throws IOException { tradeStatisticsManager = mock(TradeStatisticsManager.class); @@ -116,7 +110,6 @@ public void setup() throws IOException { dir.delete(); //noinspection ResultOfMethodCallIgnored dir.mkdir(); - keyRing = new KeyRing(new KeyStorage(dir)); } @SuppressWarnings("ConstantConditions") @@ -134,13 +127,45 @@ public void testGetCandleData() { long volume = Fiat.parseFiat("EUR", "2200").value; boolean isBullish = true; - Set set = new HashSet<>(); + Set set = new HashSet<>(); final Date now = new Date(); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "520"), Coin.parseCoin("1"), new Date(now.getTime()), null, null)); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "500"), Coin.parseCoin("1"), new Date(now.getTime() + 100), null, null)); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "600"), Coin.parseCoin("1"), new Date(now.getTime() + 200), null, null)); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "580"), Coin.parseCoin("1"), new Date(now.getTime() + 300), null, null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "520").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime(), + null, + null, + null, + null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "500").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime() + 100, + null, + null, + null, + null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "600").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime() + 200, + null, + null, + null, + null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "580").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime() + 300, + null, + null, + null, + null)); CandleData candleData = model.getCandleData(model.roundToTick(now, TradesChartsViewModel.TickUnit.DAY).getTime(), set); assertEquals(open, candleData.open); @@ -183,7 +208,6 @@ class Trade { ArrayList trades = new ArrayList<>(); // Set predetermined time to use as "now" during test - Date test_time = dateFormat.parse("2018-01-01T00:00:05"); // Monday /* new MockUp() { @Mock long currentTimeMillis() { @@ -194,11 +218,19 @@ long currentTimeMillis() { // Two trades 10 seconds apart, different YEAR, MONTH, WEEK, DAY, HOUR, MINUTE_10 trades.add(new Trade("2017-12-31T23:59:52", "1", "100", "EUR")); trades.add(new Trade("2018-01-01T00:00:02", "1", "110", "EUR")); - Set set = new HashSet<>(); + Set set = new HashSet<>(); trades.forEach(t -> - set.add(new TradeStatistics2(offer, Price.parse(t.cc, t.price), Coin.parseCoin(t.size), t.date, null, null)) + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse(t.cc, t.price).getValue(), + Coin.parseCoin(t.size).getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + t.date.getTime(), + null, + null, + null, + null)) ); - ObservableSet tradeStats = FXCollections.observableSet(set); + ObservableSet tradeStats = FXCollections.observableSet(set); // Run test for each tick type for (TradesChartsViewModel.TickUnit tick : TradesChartsViewModel.TickUnit.values()) { @@ -209,7 +241,7 @@ long currentTimeMillis() { // Trigger chart update model.setTickUnit(tick); - assertEquals(model.selectedTradeCurrencyProperty.get().getCode(), tradeStats.iterator().next().getCurrencyCode()); + assertEquals(model.selectedTradeCurrencyProperty.get().getCode(), tradeStats.iterator().next().getCurrency()); assertEquals(2, model.priceItems.size()); assertEquals(2, model.volumeItems.size()); } diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java index 19691c3d66f..c382b82cf24 100644 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -90,7 +90,8 @@ private void start() throws Throwable { Capability.DAO_STATE, Capability.BUNDLE_OF_ENVELOPES, Capability.REFUND_AGENT, - Capability.MEDIATION); + Capability.MEDIATION, + Capability.TRADE_STATISTICS_3); // assemble Metrics // - create reporters diff --git a/monitor/src/main/java/bisq/monitor/metric/MarketStats.java b/monitor/src/main/java/bisq/monitor/metric/MarketStats.java index 389722329a3..f831cafd135 100644 --- a/monitor/src/main/java/bisq/monitor/metric/MarketStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/MarketStats.java @@ -20,30 +20,15 @@ import bisq.monitor.Metric; import bisq.monitor.Reporter; -import bisq.asset.Asset; -import bisq.asset.AssetRegistry; - -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import org.berndpruenster.netlayer.tor.TorCtlException; - -import com.runjva.sourceforge.jsocks.protocol.SocksSocket; - -import java.net.HttpURLConnection; -import java.net.Socket; import java.net.URL; import java.net.URLConnection; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; @@ -110,7 +95,6 @@ protected void execute() { } catch (IllegalStateException ignore) { // no match found } catch (IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java index 189c739b3b0..0f7e8afb83c 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java @@ -23,7 +23,7 @@ import bisq.core.account.witness.AccountAgeWitnessStore; import bisq.core.offer.OfferPayload; import bisq.core.proto.persistable.CorePersistenceProtoResolver; -import bisq.core.trade.statistics.TradeStatistics2Store; +import bisq.core.trade.statistics.TradeStatistics3Store; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; @@ -163,8 +163,8 @@ public void configure(Properties properties) { String networkPostfix = "_" + BaseCurrencyNetwork.values()[Version.getBaseCurrencyNetwork()].toString(); try { PersistenceManager persistenceManager = new PersistenceManager<>(dir, new CorePersistenceProtoResolver(null, null), null); - TradeStatistics2Store tradeStatistics2Store = (TradeStatistics2Store) persistenceManager.getPersisted(TradeStatistics2Store.class.getSimpleName() + networkPostfix); - hashes.addAll(tradeStatistics2Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); + TradeStatistics3Store tradeStatistics3Store = (TradeStatistics3Store) persistenceManager.getPersisted(TradeStatistics3Store.class.getSimpleName() + networkPostfix); + hashes.addAll(tradeStatistics3Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); AccountAgeWitnessStore accountAgeWitnessStore = (AccountAgeWitnessStore) persistenceManager.getPersisted(AccountAgeWitnessStore.class.getSimpleName() + networkPostfix); hashes.addAll(accountAgeWitnessStore.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index b9fbefb5551..4a231d8da61 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -133,7 +133,7 @@ protected void execute() { networkProtoResolver); DefaultSeedNodeRepository seedNodeRepository = new DefaultSeedNodeRepository(config); PeerManager peerManager = new PeerManager(networkNode, seedNodeRepository, new ClockWatcher(), - maxConnections, new PersistenceManager<>(storageDir, persistenceProtoResolver, corruptedStorageFileHandler)); + new PersistenceManager<>(storageDir, persistenceProtoResolver, corruptedStorageFileHandler), maxConnections); // init file storage peerManager.readPersisted(); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java index e87d7468253..2ca12716f04 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java @@ -27,7 +27,7 @@ import bisq.core.dao.monitoring.network.messages.GetProposalStateHashesRequest; import bisq.core.dao.monitoring.network.messages.GetStateHashesResponse; import bisq.core.proto.persistable.CorePersistenceProtoResolver; -import bisq.core.trade.statistics.TradeStatistics2Store; +import bisq.core.trade.statistics.TradeStatistics3Store; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; @@ -137,8 +137,8 @@ public void configure(Properties properties) { String networkPostfix = "_" + BaseCurrencyNetwork.values()[Version.getBaseCurrencyNetwork()].toString(); try { PersistenceManager persistenceManager = new PersistenceManager<>(dir, new CorePersistenceProtoResolver(null, null), null); - TradeStatistics2Store tradeStatistics2Store = (TradeStatistics2Store) persistenceManager.getPersisted(TradeStatistics2Store.class.getSimpleName() + networkPostfix); - hashes.addAll(tradeStatistics2Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); + TradeStatistics3Store tradeStatistics3Store = (TradeStatistics3Store) persistenceManager.getPersisted(TradeStatistics3Store.class.getSimpleName() + networkPostfix); + hashes.addAll(tradeStatistics3Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); AccountAgeWitnessStore accountAgeWitnessStore = (AccountAgeWitnessStore) persistenceManager.getPersisted(AccountAgeWitnessStore.class.getSimpleName() + networkPostfix); hashes.addAll(accountAgeWitnessStore.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); @@ -233,9 +233,9 @@ void report() { int oldest = (int) nodeAddressTupleMap.values().stream().min(Comparator.comparingLong(Tuple::getHeight)).get().height; // - update queried height - if(type.contains("DaoState")) + if (type.contains("DaoState")) daostateheight = oldest - 20; - else if(type.contains("Proposal")) + else if (type.contains("Proposal")) proposalheight = oldest - 20; else blindvoteheight = oldest - 20; @@ -255,7 +255,6 @@ else if(type.contains("Proposal")) List states = hitcount.entrySet().stream().sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue())).map(byteBufferIntegerEntry -> byteBufferIntegerEntry.getKey()).collect(Collectors.toList()); hitcount.clear(); - nodeAddressTupleMap.forEach((nodeAddress, tuple) -> daoreport.put(type + "." + OnionParser.prettyPrint(nodeAddress) + ".hash", Integer.toString(Arrays.asList(states.toArray()).indexOf(ByteBuffer.wrap(tuple.hash))))); // - report reference head diff --git a/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java b/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java index 9258eb5e8d5..fd10691f727 100644 --- a/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java @@ -54,7 +54,7 @@ /** * Fetches fee and price data from the configured price nodes. * Based on the work of HarryMcFinned. - * + * * @author Florian Reimair * @author HarryMcFinned * @@ -159,7 +159,6 @@ protected void execute() { } } } catch (TorCtlException | IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java index 16499689b08..a40c0261662 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java @@ -21,6 +21,7 @@ import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.monitor.StatisticsHelper; + import bisq.network.p2p.NodeAddress; import org.berndpruenster.netlayer.tor.Tor; @@ -33,6 +34,7 @@ import java.util.ArrayList; import java.util.List; + import static com.google.common.base.Preconditions.checkNotNull; /** @@ -84,7 +86,6 @@ protected void execute() { reporter.report(StatisticsHelper.process(samples), getName()); } } catch (TorCtlException | IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java b/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java index 6e53e02462b..9ea3198e197 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java @@ -58,7 +58,6 @@ public void configure(Properties properties) { try { torOverrides = new Torrc(overrides); } catch (IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } @@ -79,7 +78,6 @@ protected void execute() { // stop the timer and set its timestamp reporter.report(System.currentTimeMillis() - start, getName()); } catch (TorCtlException e) { - // TODO Auto-generated catch block e.printStackTrace(); } finally { // cleanup diff --git a/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java b/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java index c3eb61e7b41..2747bcedb27 100644 --- a/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java +++ b/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java @@ -19,5 +19,5 @@ public interface DecryptedDirectMessageListener { - void onDirectMessage(DecryptedMessageWithPubKey decryptedMessageWithPubKey, @SuppressWarnings("UnusedParameters") NodeAddress peerNodeAddress); + void onDirectMessage(DecryptedMessageWithPubKey decryptedMessageWithPubKey, NodeAddress peerNodeAddress); } diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 4d2a5ef9025..8716c81d9d1 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -31,7 +31,6 @@ import bisq.network.p2p.peers.PeerManager; import bisq.network.p2p.peers.getdata.RequestDataManager; import bisq.network.p2p.peers.keepalive.KeepAliveManager; -import bisq.network.p2p.peers.peerexchange.Peer; import bisq.network.p2p.peers.peerexchange.PeerExchangeManager; import bisq.network.p2p.seed.SeedNodeRepository; import bisq.network.p2p.storage.HashMapChangedListener; @@ -46,19 +45,23 @@ import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.UserThread; +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; import bisq.common.crypto.CryptoException; import bisq.common.crypto.KeyRing; import bisq.common.crypto.PubKeyRing; import bisq.common.proto.ProtobufferException; import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.persistable.PersistedDataHost; -import bisq.common.util.Tuple2; +import bisq.common.util.Utilities; import com.google.inject.Inject; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; @@ -74,21 +77,26 @@ import java.security.PublicKey; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.UUID; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import lombok.Getter; +import lombok.Value; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -118,7 +126,7 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis private final Set decryptedMailboxListeners = new CopyOnWriteArraySet<>(); private final Set p2pServiceListeners = new CopyOnWriteArraySet<>(); @Getter - private final Map> mailboxMap = new HashMap<>(); + private final Map mailboxItemsByUid = new HashMap<>(); private final Set shutDownResultHandlers = new CopyOnWriteArraySet<>(); private final BooleanProperty hiddenServicePublished = new SimpleBooleanProperty(); private final BooleanProperty preliminaryDataReceived = new SimpleBooleanProperty(); @@ -164,7 +172,6 @@ public P2PService(NetworkNode networkNode, this.networkNode.addConnectionListener(this); this.networkNode.addMessageListener(this); - this.p2PDataStorage.addHashMapChangedListener(this); this.requestDataManager.addListener(this); // We need to have both the initial data delivered and the hidden service published @@ -197,13 +204,11 @@ public void start(@Nullable P2PServiceListener listener) { public void onAllServicesInitialized() { if (networkNode.getNodeAddress() != null) { - maybeProcessAllMailboxEntries(); myNodeAddress = networkNode.getNodeAddress(); } else { // If our HS is still not published networkNode.nodeAddressProperty().addListener((observable, oldValue, newValue) -> { if (newValue != null) { - maybeProcessAllMailboxEntries(); myNodeAddress = networkNode.getNodeAddress(); } }); @@ -280,12 +285,13 @@ public void onTorNodeReady() { boolean seedNodesAvailable = requestDataManager.requestPreliminaryData(); keepAliveManager.start(); - p2pServiceListeners.stream().forEach(SetupListener::onTorNodeReady); + p2pServiceListeners.forEach(SetupListener::onTorNodeReady); if (!seedNodesAvailable) { isBootstrapped = true; - maybeProcessAllMailboxEntries(); - p2pServiceListeners.stream().forEach(P2PServiceListener::onNoSeedNodeAvailable); + // As we do not expect a updated data request response we start here with addHashMapChangedListenerAndApply + addHashMapChangedListenerAndApply(); + p2pServiceListeners.forEach(P2PServiceListener::onNoSeedNodeAvailable); } } @@ -295,17 +301,17 @@ public void onHiddenServicePublished() { hiddenServicePublished.set(true); - p2pServiceListeners.stream().forEach(SetupListener::onHiddenServicePublished); + p2pServiceListeners.forEach(SetupListener::onHiddenServicePublished); } @Override public void onSetupFailed(Throwable throwable) { - p2pServiceListeners.stream().forEach(e -> e.onSetupFailed(throwable)); + p2pServiceListeners.forEach(e -> e.onSetupFailed(throwable)); } @Override public void onRequestCustomBridges() { - p2pServiceListeners.stream().forEach(SetupListener::onRequestCustomBridges); + p2pServiceListeners.forEach(SetupListener::onRequestCustomBridges); } // Called from networkReadyBinding @@ -317,8 +323,6 @@ private void onNetworkReady() { "seedNodeOfPreliminaryDataRequest must be present"); requestDataManager.requestUpdateData(); - /*if (Capabilities.app.containsAll(Capability.SEED_NODE)) - UserThread.runPeriodically(() -> requestDataManager.requestUpdateData(), 1, TimeUnit.HOURS);*/ // If we start up first time we don't have any peers so we need to request from seed node. // As well it can be that the persisted peer list is outdated with dead peers. @@ -346,25 +350,27 @@ public void onPreliminaryDataReceived() { public void onUpdatedDataReceived() { if (!isBootstrapped) { isBootstrapped = true; - maybeProcessAllMailboxEntries(); - p2pServiceListeners.stream().forEach(P2PServiceListener::onUpdatedDataReceived); + // Only now we start listening and processing. The p2PDataStorage is our cache for data we have received + // after the hidden service was ready. + addHashMapChangedListenerAndApply(); + p2pServiceListeners.forEach(P2PServiceListener::onUpdatedDataReceived); p2PDataStorage.onBootstrapComplete(); } } @Override public void onNoSeedNodeAvailable() { - p2pServiceListeners.stream().forEach(P2PServiceListener::onNoSeedNodeAvailable); + p2pServiceListeners.forEach(P2PServiceListener::onNoSeedNodeAvailable); } @Override public void onNoPeersAvailable() { - p2pServiceListeners.stream().forEach(P2PServiceListener::onNoPeersAvailable); + p2pServiceListeners.forEach(P2PServiceListener::onNoPeersAvailable); } @Override public void onDataReceived() { - p2pServiceListeners.stream().forEach(P2PServiceListener::onDataReceived); + p2pServiceListeners.forEach(P2PServiceListener::onDataReceived); } @@ -398,58 +404,121 @@ public void onError(Throwable throwable) { @Override public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof PrefixedSealedAndSignedMessage) { - // Seed nodes don't have set the encryptionService + PrefixedSealedAndSignedMessage sealedMsg = (PrefixedSealedAndSignedMessage) networkEnvelope; + connection.setPeerType(Connection.PeerType.DIRECT_MSG_PEER); try { - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = (PrefixedSealedAndSignedMessage) networkEnvelope; - if (verifyAddressPrefixHash(prefixedSealedAndSignedMessage)) { - // We set connectionType to that connection to avoid that is get closed when - // we get too many connection attempts. - connection.setPeerType(Connection.PeerType.DIRECT_MSG_PEER); - - log.debug("Try to decrypt..."); - DecryptedMessageWithPubKey decryptedMessageWithPubKey = encryptionService.decryptAndVerify( - prefixedSealedAndSignedMessage.getSealedAndSigned()); - - log.debug("\n\nDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD\n" + - "Decrypted SealedAndSignedMessage:\ndecryptedMsgWithPubKey={}" - + "\nDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD\n", decryptedMessageWithPubKey); - if (connection.getPeersNodeAddressOptional().isPresent()) - decryptedDirectMessageListeners.forEach( - e -> e.onDirectMessage(decryptedMessageWithPubKey, connection.getPeersNodeAddressOptional().get())); - else - log.error("peersNodeAddress is not available at onMessage."); - } else { - log.debug("Wrong receiverAddressMaskHash. The message is not intended for us."); - } + DecryptedMessageWithPubKey decryptedMsg = encryptionService.decryptAndVerify(sealedMsg.getSealedAndSigned()); + connection.getPeersNodeAddressOptional().ifPresentOrElse(nodeAddress -> + decryptedDirectMessageListeners.forEach(e -> e.onDirectMessage(decryptedMsg, nodeAddress)), + () -> { + log.error("peersNodeAddress is expected to be available at onMessage for " + + "processing PrefixedSealedAndSignedMessage."); + }); } catch (CryptoException e) { - log.debug(networkEnvelope.toString()); - log.debug(e.toString()); - log.debug("Decryption of prefixedSealedAndSignedMessage.sealedAndSigned failed. " + - "That is expected if the message is not intended for us."); + log.warn("Decryption of a direct message failed. This is not expected as the " + + "direct message was sent to our node."); } catch (ProtobufferException e) { - log.error("Protobuffer data could not be processed: {}", e.toString()); + log.error("ProtobufferException at decryptAndVerify: {}", e.toString()); + e.getStackTrace(); } } } /////////////////////////////////////////////////////////////////////////////////////////// - // HashMapChangedListener implementation + // HashMapChangedListener implementation for ProtectedStorageEntry items /////////////////////////////////////////////////////////////////////////////////////////// + private void addHashMapChangedListenerAndApply() { + p2PDataStorage.addHashMapChangedListener(this); + onAdded(p2PDataStorage.getMap().values()); + } + @Override public void onAdded(Collection protectedStorageEntries) { - protectedStorageEntries.forEach(protectedStorageEntry -> { - if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) - processMailboxEntry((ProtectedMailboxStorageEntry) protectedStorageEntry); + Collection entries = protectedStorageEntries.stream() + .filter(e -> e instanceof ProtectedMailboxStorageEntry) + .map(e -> (ProtectedMailboxStorageEntry) e) + .filter(e -> networkNode.getNodeAddress() != null) + .filter(e -> !seedNodeRepository.isSeedNode(networkNode.getNodeAddress())) // Seed nodes don't expect mailbox messages + .collect(Collectors.toSet()); + if (entries.size() > 1) { + threadedBatchProcessMailboxEntries(entries); + } else if (entries.size() == 1) { + processSingleMailboxEntry(entries); + } + } + + private void processSingleMailboxEntry(Collection protectedMailboxStorageEntries) { + checkArgument(protectedMailboxStorageEntries.size() == 1); + var decryptedEntries = new ArrayList<>(getDecryptedEntries(protectedMailboxStorageEntries)); + if (decryptedEntries.size() == 1) { + storeMailboxDataAndNotifyListeners(decryptedEntries.get(0)); + } + } + + // We run the batch processing of all mailbox messages we have received at startup in a thread to not block the UI. + // For about 1000 messages decryption takes about 1 sec. + private void threadedBatchProcessMailboxEntries(Collection protectedMailboxStorageEntries) { + ListeningExecutorService executor = Utilities.getSingleThreadListeningExecutor("processMailboxEntry-" + new Random().nextInt(1000)); + long ts = System.currentTimeMillis(); + ListenableFuture> future = executor.submit(() -> { + var decryptedEntries = getDecryptedEntries(protectedMailboxStorageEntries); + log.info("Batch processing of {} mailbox entries took {} ms", + protectedMailboxStorageEntries.size(), + System.currentTimeMillis() - ts); + return decryptedEntries; }); + + Futures.addCallback(future, new FutureCallback<>() { + public void onSuccess(Set decryptedMailboxMessageWithEntries) { + UserThread.execute(() -> decryptedMailboxMessageWithEntries.forEach(e -> storeMailboxDataAndNotifyListeners(e))); + } + + public void onFailure(@NotNull Throwable throwable) { + log.error(throwable.toString()); + } + }, MoreExecutors.directExecutor()); } - @Override - public void onRemoved(Collection protectedStorageEntries) { - // not used + private Set getDecryptedEntries(Collection protectedMailboxStorageEntries) { + Set decryptedMailboxMessageWithEntries = new HashSet<>(); + protectedMailboxStorageEntries.stream() + .map(this::decryptProtectedMailboxStorageEntry) + .filter(Objects::nonNull) + .forEach(decryptedMailboxMessageWithEntries::add); + return decryptedMailboxMessageWithEntries; } + @Nullable + private MailboxItem decryptProtectedMailboxStorageEntry(ProtectedMailboxStorageEntry protectedMailboxStorageEntry) { + try { + DecryptedMessageWithPubKey decryptedMessageWithPubKey = encryptionService.decryptAndVerify(protectedMailboxStorageEntry + .getMailboxStoragePayload() + .getPrefixedSealedAndSignedMessage() + .getSealedAndSigned()); + checkArgument(decryptedMessageWithPubKey.getNetworkEnvelope() instanceof MailboxMessage); + return new MailboxItem(protectedMailboxStorageEntry, decryptedMessageWithPubKey); + } catch (CryptoException ignore) { + // Expected if message was not intended for us + } catch (ProtobufferException e) { + log.error(e.toString()); + e.getStackTrace(); + } + return null; + } + + private void storeMailboxDataAndNotifyListeners(MailboxItem mailboxItem) { + DecryptedMessageWithPubKey decryptedMessageWithPubKey = mailboxItem.getDecryptedMessageWithPubKey(); + MailboxMessage mailboxMessage = (MailboxMessage) decryptedMessageWithPubKey.getNetworkEnvelope(); + NodeAddress sender = mailboxMessage.getSenderNodeAddress(); + mailboxItemsByUid.put(mailboxMessage.getUid(), mailboxItem); + log.info("Received a {} mailbox message with uid {} and senderAddress {}", + mailboxMessage.getClass().getSimpleName(), mailboxMessage.getUid(), sender); + decryptedMailboxListeners.forEach(e -> e.onMailboxMessageAdded(decryptedMessageWithPubKey, sender)); + } + + /////////////////////////////////////////////////////////////////////////////////////////// // DirectMessages /////////////////////////////////////////////////////////////////////////////////////////// @@ -486,13 +555,15 @@ private void doSendEncryptedDirectMessage(@NotNull NodeAddress peersNodeAddress, log.debug("\n\nEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE\n" + "Encrypt message:\nmessage={}" + "\nEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE\n", message); - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = new PrefixedSealedAndSignedMessage( - networkNode.getNodeAddress(), - encryptionService.encryptAndSign(pubKeyRing, message), - peersNodeAddress.getAddressPrefixHash(), - UUID.randomUUID().toString()); - SettableFuture future = networkNode.sendMessage(peersNodeAddress, prefixedSealedAndSignedMessage); - Futures.addCallback(future, new FutureCallback() { + + // Prefix is not needed for direct messages but as old code is doing the verification we still need to + // send it if peer has not updated. + PrefixedSealedAndSignedMessage sealedMsg = getPrefixedSealedAndSignedMessage(peersNodeAddress, + pubKeyRing, + message); + + SettableFuture future = networkNode.sendMessage(peersNodeAddress, sealedMsg); + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(@Nullable Connection connection) { sendDirectMessageListener.onArrived(); @@ -513,48 +584,30 @@ public void onFailure(@NotNull Throwable throwable) { } } + private PrefixedSealedAndSignedMessage getPrefixedSealedAndSignedMessage(NodeAddress peersNodeAddress, + PubKeyRing pubKeyRing, + NetworkEnvelope message) throws CryptoException { + byte[] addressPrefixHash; + if (peerManager.peerHasCapability(peersNodeAddress, Capability.NO_ADDRESS_PRE_FIX)) { + // The peer has an updated version so we do not need to send the prefix. + // We cannot use null as not updated nodes would get a nullPointer at protobuf serialisation. + addressPrefixHash = new byte[0]; + } else { + addressPrefixHash = peersNodeAddress.getAddressPrefixHash(); + } + return new PrefixedSealedAndSignedMessage( + networkNode.getNodeAddress(), + encryptionService.encryptAndSign(pubKeyRing, message), + addressPrefixHash, + UUID.randomUUID().toString()); + } + /////////////////////////////////////////////////////////////////////////////////////////// // MailboxMessages /////////////////////////////////////////////////////////////////////////////////////////// - private void processMailboxEntry(ProtectedMailboxStorageEntry protectedMailboxStorageEntry) { - NodeAddress nodeAddress = networkNode.getNodeAddress(); - // Seed nodes don't receive mailbox network_messages - if (nodeAddress != null && !seedNodeRepository.isSeedNode(nodeAddress)) { - MailboxStoragePayload mailboxStoragePayload = protectedMailboxStorageEntry.getMailboxStoragePayload(); - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = mailboxStoragePayload.getPrefixedSealedAndSignedMessage(); - if (verifyAddressPrefixHash(prefixedSealedAndSignedMessage)) { - try { - DecryptedMessageWithPubKey decryptedMessageWithPubKey = encryptionService.decryptAndVerify( - prefixedSealedAndSignedMessage.getSealedAndSigned()); - if (decryptedMessageWithPubKey.getNetworkEnvelope() instanceof MailboxMessage) { - MailboxMessage mailboxMessage = (MailboxMessage) decryptedMessageWithPubKey.getNetworkEnvelope(); - NodeAddress senderNodeAddress = mailboxMessage.getSenderNodeAddress(); - checkNotNull(senderNodeAddress, "senderAddress must not be null for mailbox network_messages"); - - mailboxMap.put(mailboxMessage.getUid(), new Tuple2<>(protectedMailboxStorageEntry, decryptedMessageWithPubKey)); - log.info("Received a {} mailbox message with messageUid {} and senderAddress {}", mailboxMessage.getClass().getSimpleName(), mailboxMessage.getUid(), senderNodeAddress); - decryptedMailboxListeners.forEach( - e -> e.onMailboxMessageAdded(decryptedMessageWithPubKey, senderNodeAddress)); - } else { - log.warn("tryDecryptMailboxData: Expected MailboxMessage but got other type. " + - "decryptedMsgWithPubKey.message={}", decryptedMessageWithPubKey.getNetworkEnvelope()); - } - } catch (CryptoException e) { - log.debug(e.toString()); - log.debug("Decryption of prefixedSealedAndSignedMessage.sealedAndSigned failed. " + - "That is expected if the message is not intended for us."); - } catch (ProtobufferException e) { - log.error("Protobuffer data could not be processed: {}", e.toString()); - } - } else { - log.trace("Wrong blurredAddressHash. The message is not intended for us."); - } - } - } - - public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing peersPubKeyRing, + public void sendEncryptedMailboxMessage(NodeAddress peer, PubKeyRing peersPubKeyRing, NetworkEnvelope message, SendMailboxMessageListener sendMailboxMessageListener) { if (peersPubKeyRing == null) { @@ -562,12 +615,10 @@ public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing return; } - checkNotNull(peersNodeAddress, - "PeerAddress must not be null (sendEncryptedMailboxMessage)"); + checkNotNull(peer, "PeerAddress must not be null (sendEncryptedMailboxMessage)"); checkNotNull(networkNode.getNodeAddress(), "My node address must not be null at sendEncryptedMailboxMessage"); - checkArgument(!keyRing.getPubKeyRing().equals(peersPubKeyRing), - "We got own keyring instead of that from peer"); + checkArgument(!keyRing.getPubKeyRing().equals(peersPubKeyRing), "We got own keyring instead of that from peer"); if (!isBootstrapped()) throw new NetworkNotReadyException(); @@ -578,7 +629,7 @@ public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing return; } - if (capabilityRequiredAndCapabilityNotSupported(peersNodeAddress, message)) { + if (capabilityRequiredAndCapabilityNotSupported(peer, message)) { sendMailboxMessageListener.onFault("We did not send the EncryptedMailboxMessage " + "because the peer does not support the capability."); return; @@ -589,15 +640,12 @@ public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing "Encrypt message:\nmessage={}" + "\nEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE\n", message); - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = new PrefixedSealedAndSignedMessage( - networkNode.getNodeAddress(), - encryptionService.encryptAndSign(peersPubKeyRing, message), - peersNodeAddress.getAddressPrefixHash(), - UUID.randomUUID().toString()); + PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = getPrefixedSealedAndSignedMessage(peer, + peersPubKeyRing, message); - log.debug("sendEncryptedMailboxMessage msg={}, peersNodeAddress={}", message, peersNodeAddress); - SettableFuture future = networkNode.sendMessage(peersNodeAddress, prefixedSealedAndSignedMessage); - Futures.addCallback(future, new FutureCallback() { + log.debug("sendEncryptedMailboxMessage msg={}, peersNodeAddress={}", message, peer); + SettableFuture future = networkNode.sendMessage(peer, prefixedSealedAndSignedMessage); + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(@Nullable Connection connection) { sendMailboxMessageListener.onArrived(); @@ -624,22 +672,11 @@ private boolean capabilityRequiredAndCapabilityNotSupported(NodeAddress peersNod if (!(message instanceof CapabilityRequiringPayload)) return false; - // We only expect AckMessage so far - if (!(message instanceof AckMessage)) - log.warn("We got a CapabilityRequiringPayload for the mailbox message which is not a AckMessage. " + - "peersNodeAddress={}", peersNodeAddress); - - Set allPeers = peerManager.getPersistedPeers(); - allPeers.addAll(peerManager.getReportedPeers()); - allPeers.addAll(peerManager.getLivePeers(null)); // We might have multiple entries of the same peer without the supportedCapabilities field set if we received // it from old versions, so we filter those. - Optional optionalPeer = allPeers.stream() - .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) - .filter(peer -> !peer.getCapabilities().isEmpty()) - .findAny(); - if (optionalPeer.isPresent()) { - boolean result = optionalPeer.get().getCapabilities().containsAll(((CapabilityRequiringPayload) message).getRequiredCapabilities()); + Optional optionalCapabilities = peerManager.findPeersCapabilities(peersNodeAddress); + if (optionalCapabilities.isPresent()) { + boolean result = optionalCapabilities.get().containsAll(((CapabilityRequiringPayload) message).getRequiredCapabilities()); if (!result) log.warn("We don't send the message because the peer does not support the required capability. " + @@ -654,15 +691,6 @@ private boolean capabilityRequiredAndCapabilityNotSupported(NodeAddress peersNod } - private void maybeProcessAllMailboxEntries() { - if (isBootstrapped) { - p2PDataStorage.getMap().values().forEach(protectedStorageEntry -> { - if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) - processMailboxEntry((ProtectedMailboxStorageEntry) protectedStorageEntry); - }); - } - } - private void addMailboxData(MailboxStoragePayload expirableMailboxStoragePayload, PublicKey receiversPublicKey, SendMailboxMessageListener sendMailboxMessageListener) { @@ -742,8 +770,8 @@ private void delayedRemoveEntryFromMailbox(DecryptedMessageWithPubKey decryptedM MailboxMessage mailboxMessage = (MailboxMessage) decryptedMessageWithPubKey.getNetworkEnvelope(); String uid = mailboxMessage.getUid(); - if (mailboxMap.containsKey(uid)) { - ProtectedMailboxStorageEntry mailboxData = mailboxMap.get(uid).first; + if (mailboxItemsByUid.containsKey(uid)) { + ProtectedMailboxStorageEntry mailboxData = mailboxItemsByUid.get(uid).getProtectedMailboxStorageEntry(); if (mailboxData != null && mailboxData.getProtectedStoragePayload() instanceof MailboxStoragePayload) { MailboxStoragePayload expirableMailboxStoragePayload = (MailboxStoragePayload) mailboxData.getProtectedStoragePayload(); PublicKey receiversPubKey = mailboxData.getReceiversPubKey(); @@ -759,7 +787,7 @@ private void delayedRemoveEntryFromMailbox(DecryptedMessageWithPubKey decryptedM log.error("Signing at getDataWithSignedSeqNr failed. That should never happen."); } - mailboxMap.remove(uid); + mailboxItemsByUid.remove(uid); log.info("Removed successfully decryptedMsgWithPubKey. uid={}", uid); } } else { @@ -840,8 +868,7 @@ public void addP2PServiceListener(P2PServiceListener listener) { } public void removeP2PServiceListener(P2PServiceListener listener) { - if (p2pServiceListeners.contains(listener)) - p2pServiceListeners.remove(listener); + p2pServiceListeners.remove(listener); } public void addHashSetChangedListener(HashMapChangedListener hashMapChangedListener) { @@ -893,18 +920,23 @@ public KeyRing getKeyRing() { return keyRing; } - /////////////////////////////////////////////////////////////////////////////////////////// - // Private - /////////////////////////////////////////////////////////////////////////////////////////// + public Optional findPeersCapabilities(NodeAddress peer) { + return networkNode.getConfirmedConnections().stream() + .filter(e -> e.getPeersNodeAddressOptional().isPresent()) + .filter(e -> e.getPeersNodeAddressOptional().get().equals(peer)) + .map(Connection::getCapabilities) + .findAny(); + } - private boolean verifyAddressPrefixHash(PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage) { - if (networkNode.getNodeAddress() != null) { - byte[] blurredAddressHash = networkNode.getNodeAddress().getAddressPrefixHash(); - return blurredAddressHash != null && - Arrays.equals(blurredAddressHash, prefixedSealedAndSignedMessage.getAddressPrefixHash()); - } else { - log.debug("myOnionAddress is null at verifyAddressPrefixHash. That is expected at startup."); - return false; + @Value + public class MailboxItem { + private final ProtectedMailboxStorageEntry protectedMailboxStorageEntry; + private final DecryptedMessageWithPubKey decryptedMessageWithPubKey; + + public MailboxItem(ProtectedMailboxStorageEntry protectedMailboxStorageEntry, + DecryptedMessageWithPubKey decryptedMessageWithPubKey) { + this.protectedMailboxStorageEntry = protectedMailboxStorageEntry; + this.decryptedMessageWithPubKey = decryptedMessageWithPubKey; } } } diff --git a/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java b/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java index d5f626a90d5..bf617dce114 100644 --- a/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java +++ b/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java @@ -33,7 +33,11 @@ public final class PrefixedSealedAndSignedMessage extends NetworkEnvelope implements MailboxMessage, SendersNodeAddressMessage { private final NodeAddress senderNodeAddress; private final SealedAndSigned sealedAndSigned; + + // From v1.4.0 on addressPrefixHash can be an empty byte array. + // We cannot make it nullable as not updated nodes would get a nullPointer exception at protobuf serialisation. private final byte[] addressPrefixHash; + private final String uid; public PrefixedSealedAndSignedMessage(NodeAddress senderNodeAddress, @@ -71,7 +75,8 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { .build(); } - public static PrefixedSealedAndSignedMessage fromProto(protobuf.PrefixedSealedAndSignedMessage proto, int messageVersion) { + public static PrefixedSealedAndSignedMessage fromProto(protobuf.PrefixedSealedAndSignedMessage proto, + int messageVersion) { return new PrefixedSealedAndSignedMessage(NodeAddress.fromProto(proto.getNodeAddress()), SealedAndSigned.fromProto(proto.getSealedAndSigned()), proto.getAddressPrefixHash().toByteArray(), diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index 8b3560e3362..dca03c133de 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -159,6 +159,8 @@ public static int getPermittedMessageSize() { private final List messageTimeStamps = new ArrayList<>(); private final CopyOnWriteArraySet messageListeners = new CopyOnWriteArraySet<>(); private volatile long lastSendTimeStamp = 0; + // We use a weak reference here to ensure that no connection causes a memory leak in case it get closed without + // the shutDown being called. private final CopyOnWriteArraySet> capabilitiesListeners = new CopyOnWriteArraySet<>(); @Getter @@ -514,6 +516,8 @@ private void doShutDown(CloseConnectionReason closeConnectionReason, @Nullable R } finally { protoOutputStream.onConnectionShutdown(); + capabilitiesListeners.clear(); + try { protoInputStream.close(); } catch (IOException e) { @@ -559,7 +563,6 @@ public String toString() { '}'; } - @SuppressWarnings("unused") public String printDetails() { String portInfo; if (socket.getLocalPort() == 0) @@ -783,30 +786,9 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { } if (networkEnvelope instanceof SupportedCapabilitiesMessage) { - Capabilities supportedCapabilities = ((SupportedCapabilitiesMessage) networkEnvelope).getSupportedCapabilities(); - if (supportedCapabilities != null) { - if (!capabilities.equals(supportedCapabilities)) { - capabilities.set(supportedCapabilities); - - // Capabilities can be empty. We only check for mandatory if we get some capabilities. - if (!capabilities.isEmpty() && !Capabilities.hasMandatoryCapability(capabilities)) { - String senderNodeAddress = networkEnvelope instanceof SendersNodeAddressMessage ? - ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress().getFullAddress() : - "[unknown address]"; - log.info("We close a connection to old node {}. " + - "Capabilities of old node: {}, networkEnvelope class name={}", - senderNodeAddress, capabilities.prettyPrint(), networkEnvelope.getClass().getSimpleName()); - shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); - return; - } - - capabilitiesListeners.forEach(weakListener -> { - SupportedCapabilitiesListener supportedCapabilitiesListener = weakListener.get(); - if (supportedCapabilitiesListener != null) { - UserThread.execute(() -> supportedCapabilitiesListener.onChanged(supportedCapabilities)); - } - }); - } + boolean causedShutDown = handleSupportedCapabilitiesMessage(networkEnvelope); + if (causedShutDown) { + return; } } @@ -882,4 +864,50 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { handleException(t); } } + + protected boolean handleSupportedCapabilitiesMessage(NetworkEnvelope networkEnvelope) { + Capabilities supportedCapabilities = ((SupportedCapabilitiesMessage) networkEnvelope).getSupportedCapabilities(); + if (supportedCapabilities == null || supportedCapabilities.isEmpty()) { + return false; + } + + if (this.capabilities.equals(supportedCapabilities)) { + return false; + } + + if (!Capabilities.hasMandatoryCapability(supportedCapabilities)) { + log.info("We close a connection because of " + + "CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED " + + "to node {}. Capabilities of old node: {}, " + + "networkEnvelope class name={}", + getSenderNodeAddressAsString(networkEnvelope), + supportedCapabilities.prettyPrint(), + networkEnvelope.getClass().getSimpleName()); + shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); + return true; + } + + this.capabilities.set(supportedCapabilities); + + capabilitiesListeners.forEach(weakListener -> { + SupportedCapabilitiesListener supportedCapabilitiesListener = weakListener.get(); + if (supportedCapabilitiesListener != null) { + UserThread.execute(() -> supportedCapabilitiesListener.onChanged(supportedCapabilities)); + } + }); + return false; + } + + @Nullable + private NodeAddress getSenderNodeAddress(NetworkEnvelope networkEnvelope) { + return getPeersNodeAddressOptional().orElse( + networkEnvelope instanceof SendersNodeAddressMessage ? + ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress() : + null); + } + + private String getSenderNodeAddressAsString(NetworkEnvelope networkEnvelope) { + NodeAddress nodeAddress = getSenderNodeAddress(networkEnvelope); + return nodeAddress == null ? "null" : nodeAddress.getFullAddress(); + } } diff --git a/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java b/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java index 5b81cbb1c98..c2695833f7a 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java +++ b/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java @@ -21,6 +21,7 @@ import bisq.common.Timer; import bisq.common.UserThread; +import bisq.common.app.Capabilities; import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.network.NetworkProtoResolver; import bisq.common.util.Utilities; @@ -496,4 +497,12 @@ private void printInboundConnections() { public NodeAddress getNodeAddress() { return nodeAddressProperty.get(); } + + public Optional findPeersCapabilities(NodeAddress nodeAddress) { + return getConfirmedConnections().stream() + .filter(c -> c.getPeersNodeAddressProperty().get() != null) + .filter(c -> c.getPeersNodeAddressProperty().get().equals(nodeAddress)) + .map(Connection::getCapabilities) + .findAny(); + } } diff --git a/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java b/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java index 4614e20f79b..96d944c75fa 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java +++ b/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java @@ -50,11 +50,11 @@ void writeEnvelope(NetworkEnvelope envelope) { } catch (InterruptedException e) { Thread currentThread = Thread.currentThread(); currentThread.interrupt(); - final String msg = "Thread " + currentThread + " was interrupted. InterruptedException=" + e; + String msg = "Thread " + currentThread + " was interrupted. InterruptedException=" + e; log.error(msg); throw new BisqRuntimeException(msg, e); } catch (ExecutionException e) { - final String msg = "Failed to write envelope. ExecutionException " + e; + String msg = "Failed to write envelope. ExecutionException " + e; log.error(msg); throw new BisqRuntimeException(msg, e); } @@ -65,7 +65,7 @@ void onConnectionShutdown() { executorService.shutdownNow(); super.onConnectionShutdown(); } catch (Throwable t) { - log.error("Failed to handle connection shutdown. Throwable={}", t); + log.error("Failed to handle connection shutdown. Throwable={}", t.toString()); } } } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index e8cf818336d..20af0453194 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -32,6 +32,7 @@ import bisq.common.Timer; import bisq.common.UserThread; import bisq.common.app.Capabilities; +import bisq.common.app.Capability; import bisq.common.config.Config; import bisq.common.persistence.PersistenceManager; import bisq.common.proto.persistable.PersistedDataHost; @@ -42,6 +43,7 @@ import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; import java.util.Date; import java.util.HashSet; @@ -59,8 +61,10 @@ import javax.annotation.Nullable; +import static com.google.common.base.Preconditions.checkArgument; + @Slf4j -public class PeerManager implements ConnectionListener, PersistedDataHost { +public final class PeerManager implements ConnectionListener, PersistedDataHost { /////////////////////////////////////////////////////////////////////////////////////////// // Static @@ -77,9 +81,6 @@ public class PeerManager implements ConnectionListener, PersistedDataHost { // Age of what we consider connected peers still as live peers private static final long MAX_AGE_LIVE_PEERS = TimeUnit.MINUTES.toMillis(30); private static final boolean PRINT_REPORTED_PEERS_DETAILS = true; - @Setter - private boolean allowDisconnectSeedNodes; - private Set latestLivePeers = new HashSet<>(); /////////////////////////////////////////////////////////////////////////////////////////// @@ -101,19 +102,23 @@ public interface Listener { private final NetworkNode networkNode; private final ClockWatcher clockWatcher; - - private int maxConnections; private final Set seedNodeAddresses; - private final PersistenceManager persistenceManager; + private final ClockWatcher.Listener clockWatcherListener; + private final List listeners = new CopyOnWriteArrayList<>(); + + // Persistable peerList private final PeerList peerList = new PeerList(); - private final HashSet persistedPeers = new HashSet<>(); + // Peers we got reported from other peers + @Getter private final Set reportedPeers = new HashSet<>(); - private final ClockWatcher.Listener listener; - private final List listeners = new CopyOnWriteArrayList<>(); + // Most recent peers with activity date of last 30 min. + private final Set latestLivePeers = new HashSet<>(); + private Timer checkMaxConnectionsTimer; private boolean stopped; private boolean lostAllConnections; + private int maxConnections; @Getter private int minConnections; @@ -121,6 +126,8 @@ public interface Listener { private int maxConnectionsPeer; private int maxConnectionsNonDirect; private int maxConnectionsAbsolute; + @Setter + private boolean allowDisconnectSeedNodes; /////////////////////////////////////////////////////////////////////////////////////////// @@ -131,8 +138,8 @@ public interface Listener { public PeerManager(NetworkNode networkNode, SeedNodeRepository seedNodeRepository, ClockWatcher clockWatcher, - @Named(Config.MAX_CONNECTIONS) int maxConnections, - PersistenceManager persistenceManager) { + PersistenceManager persistenceManager, + @Named(Config.MAX_CONNECTIONS) int maxConnections) { this.networkNode = networkNode; this.seedNodeAddresses = new HashSet<>(seedNodeRepository.getSeedNodeAddresses()); this.clockWatcher = clockWatcher; @@ -144,7 +151,7 @@ public PeerManager(NetworkNode networkNode, setConnectionLimits(maxConnections); // we check if app was idle for more then 5 sec. - listener = new ClockWatcher.Listener() { + clockWatcherListener = new ClockWatcher.Listener() { @Override public void onSecondTick() { } @@ -155,56 +162,34 @@ public void onMinuteTick() { @Override public void onAwakeFromStandby(long missedMs) { - // TODO is "stopped = false;" correct? + // We got probably stopped set to true when we got a longer interruption (e.g. lost all connections), + // now we get awake again, so set stopped to false. stopped = false; listeners.forEach(Listener::onAwakeFromStandby); } }; - clockWatcher.addListener(listener); + clockWatcher.addListener(clockWatcherListener); } public void shutDown() { networkNode.removeConnectionListener(this); - clockWatcher.removeListener(listener); + clockWatcher.removeListener(clockWatcherListener); stopCheckMaxConnectionsTimer(); } /////////////////////////////////////////////////////////////////////////////////////////// - // API + // PersistedDataHost implementation /////////////////////////////////////////////////////////////////////////////////////////// @Override public void readPersisted() { PeerList persisted = persistenceManager.getPersisted(); if (persisted != null) { - this.persistedPeers.addAll(persisted.getList()); + peerList.setAll(persisted.getSet()); } } - public int getMaxConnections() { - return maxConnectionsAbsolute; - } - - public void addListener(Listener listener) { - listeners.add(listener); - } - - public void removeListener(Listener listener) { - listeners.remove(listener); - } - - // Modify this to change the relationships between connection limits. - // maxConnections default 12 - private void setConnectionLimits(int maxConnections) { - this.maxConnections = maxConnections; // app node 12; seedNode 30 - minConnections = Math.max(1, (int) Math.round(maxConnections * 0.7)); // app node 1-8; seedNode 21 - disconnectFromSeedNode = maxConnections; // app node 12; seedNode 30 - maxConnectionsPeer = Math.max(4, (int) Math.round(maxConnections * 1.3)); // app node 16; seedNode 39 - maxConnectionsNonDirect = Math.max(8, (int) Math.round(maxConnections * 1.7)); // app node 20; seedNode 51 - maxConnectionsAbsolute = Math.max(12, (int) Math.round(maxConnections * 2.5)); // app node 30; seedNode 66 - } - /////////////////////////////////////////////////////////////////////////////////////////// // ConnectionListener implementation @@ -212,64 +197,238 @@ private void setConnectionLimits(int maxConnections) { @Override public void onConnection(Connection connection) { - boolean seedNode = isSeedNode(connection); - Optional addressOptional = connection.getPeersNodeAddressOptional(); - if (log.isDebugEnabled()) { - String peer = addressOptional.map(NodeAddress::getFullAddress).orElseGet(() -> - "not known yet (connection id=" + connection.getUid() + ")"); - log.debug("onConnection: peer = {}{}", - peer, - seedNode ? " (SeedNode)" : ""); - } - - if (seedNode) + if (isSeedNode(connection)) { connection.setPeerType(Connection.PeerType.SEED_NODE); + } doHouseKeeping(); if (lostAllConnections) { lostAllConnections = false; stopped = false; - listeners.stream().forEach(Listener::onNewConnectionAfterAllConnectionsLost); + listeners.forEach(Listener::onNewConnectionAfterAllConnectionsLost); } + connection.getPeersNodeAddressOptional() + .flatMap(this::findPeer) + .ifPresent(Peer::onConnection); } @Override public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { - log.info("onDisconnect called: nodeAddress={}, closeConnectionReason={}", connection.getPeersNodeAddressOptional(), closeConnectionReason); - - final Optional addressOptional = connection.getPeersNodeAddressOptional(); - log.debug("onDisconnect: peer = {}{} / closeConnectionReason: {}", - (addressOptional.isPresent() ? addressOptional.get().getFullAddress() : "not known yet (connection id=" + connection.getUid() + ")"), - isSeedNode(connection) ? " (SeedNode)" : "", - closeConnectionReason); - + log.info("onDisconnect called: nodeAddress={}, closeConnectionReason={}", + connection.getPeersNodeAddressOptional(), closeConnectionReason); handleConnectionFault(connection); - lostAllConnections = networkNode.getAllConnections().isEmpty(); if (lostAllConnections) { stopped = true; log.warn("\n------------------------------------------------------------\n" + "All connections lost\n" + "------------------------------------------------------------"); - listeners.stream().forEach(Listener::onAllConnectionsLost); + listeners.forEach(Listener::onAllConnectionsLost); } + maybeRemoveBannedPeer(closeConnectionReason, connection); + } + + @Override + public void onError(Throwable throwable) { + } - if (connection.getPeersNodeAddressOptional().isPresent() && isNodeBanned(closeConnectionReason, connection)) { - final NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); + + /////////////////////////////////////////////////////////////////////////////////////////// + // Connection + /////////////////////////////////////////////////////////////////////////////////////////// + + public boolean hasSufficientConnections() { + return networkNode.getConfirmedConnections().size() >= minConnections; + } + + // Checks if that connection has the peers node address + public boolean isConfirmed(NodeAddress nodeAddress) { + return networkNode.getNodeAddressesOfConfirmedConnections().contains(nodeAddress); + } + + public void handleConnectionFault(Connection connection) { + connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> handleConnectionFault(nodeAddress, connection)); + } + + public void handleConnectionFault(NodeAddress nodeAddress) { + handleConnectionFault(nodeAddress, null); + } + + public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection connection) { + boolean doRemovePersistedPeer = false; + removeReportedPeer(nodeAddress); + Optional persistedPeerOptional = findPersistedPeer(nodeAddress); + if (persistedPeerOptional.isPresent()) { + Peer persistedPeer = persistedPeerOptional.get(); + persistedPeer.onDisconnect(); + doRemovePersistedPeer = persistedPeer.tooManyFailedConnectionAttempts(); + } + boolean ruleViolation = connection != null && connection.getRuleViolation() != null; + doRemovePersistedPeer = doRemovePersistedPeer || ruleViolation; + + if (doRemovePersistedPeer) + removePersistedPeer(nodeAddress); + else + removeTooOldPersistedPeers(); + } + + public boolean isSeedNode(Connection connection) { + return connection.getPeersNodeAddressOptional().isPresent() && + seedNodeAddresses.contains(connection.getPeersNodeAddressOptional().get()); + } + + public boolean isSelf(NodeAddress nodeAddress) { + return nodeAddress.equals(networkNode.getNodeAddress()); + } + + private boolean isSeedNode(Peer peer) { + return seedNodeAddresses.contains(peer.getNodeAddress()); + } + + public boolean isSeedNode(NodeAddress nodeAddress) { + return seedNodeAddresses.contains(nodeAddress); + } + + public boolean isPeerBanned(CloseConnectionReason closeConnectionReason, Connection connection) { + return closeConnectionReason == CloseConnectionReason.PEER_BANNED && + connection.getPeersNodeAddressOptional().isPresent(); + } + + private void maybeRemoveBannedPeer(CloseConnectionReason closeConnectionReason, Connection connection) { + if (connection.getPeersNodeAddressOptional().isPresent() && isPeerBanned(closeConnectionReason, connection)) { + NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); seedNodeAddresses.remove(nodeAddress); removePersistedPeer(nodeAddress); removeReportedPeer(nodeAddress); } } - public boolean isNodeBanned(CloseConnectionReason closeConnectionReason, Connection connection) { - return closeConnectionReason == CloseConnectionReason.PEER_BANNED && - connection.getPeersNodeAddressOptional().isPresent(); + + /////////////////////////////////////////////////////////////////////////////////////////// + // Peer + /////////////////////////////////////////////////////////////////////////////////////////// + + @SuppressWarnings("unused") + public Optional findPeer(NodeAddress peersNodeAddress) { + return getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) + .findAny(); } - @Override - public void onError(Throwable throwable) { + public Set getAllPeers() { + Set allPeers = new HashSet<>(getLivePeers()); + allPeers.addAll(getPersistedPeers()); + allPeers.addAll(reportedPeers); + return allPeers; + } + + public Collection getPersistedPeers() { + return peerList.getSet(); + } + + public void addToReportedPeers(Set reportedPeersToAdd, + Connection connection, + Capabilities capabilities) { + applyCapabilities(connection, capabilities); + + Set peers = reportedPeersToAdd.stream() + .filter(peer -> !isSelf(peer.getNodeAddress())) + .collect(Collectors.toSet()); + + printNewReportedPeers(peers); + + // We check if the reported msg is not violating our rules + if (peers.size() <= (MAX_REPORTED_PEERS + maxConnectionsAbsolute + 10)) { + reportedPeers.addAll(peers); + purgeReportedPeersIfExceeds(); + + getPersistedPeers().addAll(peers); + purgePersistedPeersIfExceeds(); + requestPersistence(); + + printReportedPeers(); + } else { + // If a node is trying to send too many list we treat it as rule violation. + // Reported list include the connected list. We use the max value and give some extra headroom. + // Will trigger a shutdown after 2nd time sending too much + connection.reportInvalidRequest(RuleViolation.TOO_MANY_REPORTED_PEERS_SENT); + } + } + + // Delivers the live peers from the last 30 min (MAX_AGE_LIVE_PEERS) + // We include older peers to avoid risks for network partitioning + public Set getLivePeers() { + return getLivePeers(null); + } + + public Set getLivePeers(@Nullable NodeAddress excludedNodeAddress) { + int oldNumLatestLivePeers = latestLivePeers.size(); + + Set peers = new HashSet<>(latestLivePeers); + Set currentLivePeers = getConnectedReportedPeers().stream() + .filter(e -> !isSeedNode(e)) + .filter(e -> !e.getNodeAddress().equals(excludedNodeAddress)) + .collect(Collectors.toSet()); + peers.addAll(currentLivePeers); + + long maxAge = new Date().getTime() - MAX_AGE_LIVE_PEERS; + latestLivePeers.clear(); + Set recentPeers = peers.stream() + .filter(peer -> peer.getDateAsLong() > maxAge) + .collect(Collectors.toSet()); + latestLivePeers.addAll(recentPeers); + + if (oldNumLatestLivePeers != latestLivePeers.size()) + log.info("Num of latestLivePeers={}", latestLivePeers.size()); + return latestLivePeers; + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Capabilities + /////////////////////////////////////////////////////////////////////////////////////////// + + public boolean peerHasCapability(NodeAddress peersNodeAddress, Capability capability) { + return findPeersCapabilities(peersNodeAddress) + .map(capabilities -> capabilities.contains(capability)) + .orElse(false); + } + + public Optional findPeersCapabilities(NodeAddress nodeAddress) { + // We look up first our connections as that is our own data. If not found there we look up the peers which + // include reported peers. + Optional optionalCapabilities = networkNode.findPeersCapabilities(nodeAddress); + if (optionalCapabilities.isPresent() && !optionalCapabilities.get().isEmpty()) { + return optionalCapabilities; + } + + // Reported peers are not trusted data. We could get capabilities which miss the + // peers real capability or we could get maliciously altered capabilities telling us the peer supports a + // capability which is in fact not supported. This could lead to connection loss as we might send data not + // recognized by the peer. As we register a listener on connection if we don't have set the capability from our + // own sources we would get it fixed as soon we have a connection with that peer, rendering such an attack + // inefficient. + // Also this risk is only for not updated peers, so in case that would be abused for an + // attack all users have a strong incentive to update ;-). + return getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) + .findAny() + .map(Peer::getCapabilities); + } + + private void applyCapabilities(Connection connection, Capabilities newCapabilities) { + if (newCapabilities == null || newCapabilities.isEmpty()) { + return; + } + + connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { + getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) + .filter(peer -> peer.getCapabilities().hasLess(newCapabilities)) + .forEach(peer -> peer.setCapabilities(newCapabilities)); + }); + requestPersistence(); } @@ -410,30 +569,21 @@ private void removeSuperfluousSeedNodes() { } } - /////////////////////////////////////////////////////////////////////////////////////////// // Reported peers /////////////////////////////////////////////////////////////////////////////////////////// - private boolean removeReportedPeer(Peer reportedPeer) { - boolean contained = reportedPeers.remove(reportedPeer); + private void removeReportedPeer(Peer reportedPeer) { + reportedPeers.remove(reportedPeer); printReportedPeers(); - return contained; } - @SuppressWarnings("UnusedReturnValue") - @Nullable - private Peer removeReportedPeer(NodeAddress nodeAddress) { + private void removeReportedPeer(NodeAddress nodeAddress) { List reportedPeersClone = new ArrayList<>(reportedPeers); - Optional reportedPeerOptional = reportedPeersClone.stream() - .filter(e -> e.getNodeAddress().equals(nodeAddress)).findAny(); - if (reportedPeerOptional.isPresent()) { - Peer reportedPeer = reportedPeerOptional.get(); - removeReportedPeer(reportedPeer); - return reportedPeer; - } else { - return null; - } + reportedPeersClone.stream() + .filter(e -> e.getNodeAddress().equals(nodeAddress)) + .findAny() + .ifPresent(this::removeReportedPeer); } private void removeTooOldReportedPeers() { @@ -444,31 +594,6 @@ private void removeTooOldReportedPeers() { reportedPeersToRemove.forEach(this::removeReportedPeer); } - public Set getReportedPeers() { - return reportedPeers; - } - - public void addToReportedPeers(Set reportedPeersToAdd, Connection connection) { - printNewReportedPeers(reportedPeersToAdd); - - // We check if the reported msg is not violating our rules - if (reportedPeersToAdd.size() <= (MAX_REPORTED_PEERS + maxConnectionsAbsolute + 10)) { - reportedPeers.addAll(reportedPeersToAdd); - purgeReportedPeersIfExceeds(); - - persistedPeers.addAll(reportedPeersToAdd); - purgePersistedPeersIfExceeds(); - peerList.setAll(persistedPeers); - persistenceManager.requestPersistence(); - - printReportedPeers(); - } else { - // If a node is trying to send too many list we treat it as rule violation. - // Reported list include the connected list. We use the max value and give some extra headroom. - // Will trigger a shutdown after 2nd time sending too much - connection.reportInvalidRequest(RuleViolation.TOO_MANY_REPORTED_PEERS_SENT); - } - } private void purgeReportedPeersIfExceeds() { int size = reportedPeers.size(); @@ -477,7 +602,7 @@ private void purgeReportedPeersIfExceeds() { "We remove random peers from the reported peers list.", size, MAX_REPORTED_PEERS); int diff = size - MAX_REPORTED_PEERS; List list = new ArrayList<>(reportedPeers); - // we dont use sorting by lastActivityDate to keep it more random + // we don't use sorting by lastActivityDate to keep it more random for (int i = 0; i < diff; i++) { if (!list.isEmpty()) { Peer toRemove = list.remove(new Random().nextInt(list.size())); @@ -491,12 +616,11 @@ private void purgeReportedPeersIfExceeds() { private void printReportedPeers() { if (!reportedPeers.isEmpty()) { - //noinspection ConstantConditions if (PRINT_REPORTED_PEERS_DETAILS) { StringBuilder result = new StringBuilder("\n\n------------------------------------------------------------\n" + "Collected reported peers:"); List reportedPeersClone = new ArrayList<>(reportedPeers); - reportedPeersClone.stream().forEach(e -> result.append("\n").append(e)); + reportedPeersClone.forEach(e -> result.append("\n").append(e)); result.append("\n------------------------------------------------------------\n"); log.trace(result.toString()); } @@ -505,11 +629,10 @@ private void printReportedPeers() { } private void printNewReportedPeers(Set reportedPeers) { - //noinspection ConstantConditions if (PRINT_REPORTED_PEERS_DETAILS) { StringBuilder result = new StringBuilder("We received new reportedPeers:"); List reportedPeersClone = new ArrayList<>(reportedPeers); - reportedPeersClone.stream().forEach(e -> result.append("\n\t").append(e)); + reportedPeersClone.forEach(e -> result.append("\n\t").append(e)); log.trace(result.toString()); } log.debug("Number of new arrived reported peers: {}", reportedPeers.size()); @@ -517,47 +640,51 @@ private void printNewReportedPeers(Set reportedPeers) { /////////////////////////////////////////////////////////////////////////////////////////// - // Persisted list + // Persisted peers /////////////////////////////////////////////////////////////////////////////////////////// private boolean removePersistedPeer(Peer persistedPeer) { - if (persistedPeers.contains(persistedPeer)) { - persistedPeers.remove(persistedPeer); - peerList.setAll(persistedPeers); - persistenceManager.requestPersistence(); + if (getPersistedPeers().contains(persistedPeer)) { + getPersistedPeers().remove(persistedPeer); + requestPersistence(); return true; } else { return false; } } + private void requestPersistence() { + persistenceManager.requestPersistence(); + } + @SuppressWarnings("UnusedReturnValue") private boolean removePersistedPeer(NodeAddress nodeAddress) { - Optional persistedPeerOptional = getPersistedPeerOptional(nodeAddress); - return persistedPeerOptional.isPresent() && removePersistedPeer(persistedPeerOptional.get()); + Optional optionalPersistedPeer = findPersistedPeer(nodeAddress); + return optionalPersistedPeer.isPresent() && removePersistedPeer(optionalPersistedPeer.get()); } - private Optional getPersistedPeerOptional(NodeAddress nodeAddress) { - return persistedPeers.stream() - .filter(e -> e.getNodeAddress().equals(nodeAddress)).findAny(); + private Optional findPersistedPeer(NodeAddress nodeAddress) { + return getPersistedPeers().stream() + .filter(e -> e.getNodeAddress().equals(nodeAddress)) + .findAny(); } private void removeTooOldPersistedPeers() { - Set persistedPeersToRemove = persistedPeers.stream() + Set persistedPeersToRemove = getPersistedPeers().stream() .filter(reportedPeer -> new Date().getTime() - reportedPeer.getDate().getTime() > MAX_AGE) .collect(Collectors.toSet()); persistedPeersToRemove.forEach(this::removePersistedPeer); } private void purgePersistedPeersIfExceeds() { - int size = persistedPeers.size(); + int size = getPersistedPeers().size(); int limit = MAX_PERSISTED_PEERS; if (size > limit) { log.trace("We have already {} persisted peers which exceeds our limit of {}." + "We remove random peers from the persisted peers list.", size, limit); int diff = size - limit; - List list = new ArrayList<>(persistedPeers); - // we dont use sorting by lastActivityDate to avoid attack vectors and keep it more random + List list = new ArrayList<>(getPersistedPeers()); + // we don't use sorting by lastActivityDate to avoid attack vectors and keep it more random for (int i = 0; i < diff; i++) { if (!list.isEmpty()) { Peer toRemove = list.remove(new Random().nextInt(list.size())); @@ -569,110 +696,44 @@ private void purgePersistedPeersIfExceeds() { } } - public Set getPersistedPeers() { - return persistedPeers; - } - /////////////////////////////////////////////////////////////////////////////////////////// - // Misc + // Getters /////////////////////////////////////////////////////////////////////////////////////////// - public boolean hasSufficientConnections() { - return networkNode.getNodeAddressesOfConfirmedConnections().size() >= minConnections; - } - - private boolean isSeedNode(Peer reportedPeer) { - return seedNodeAddresses.contains(reportedPeer.getNodeAddress()); - } - - public boolean isSeedNode(NodeAddress nodeAddress) { - return seedNodeAddresses.contains(nodeAddress); - } - - public boolean isSeedNode(Connection connection) { - return connection.hasPeersNodeAddress() && seedNodeAddresses.contains(connection.getPeersNodeAddressOptional().get()); - } - - public boolean isSelf(Peer reportedPeer) { - return isSelf(reportedPeer.getNodeAddress()); - } - - public boolean isSelf(NodeAddress nodeAddress) { - return nodeAddress.equals(networkNode.getNodeAddress()); - } - - public boolean isConfirmed(Peer reportedPeer) { - return isConfirmed(reportedPeer.getNodeAddress()); - } - - // Checks if that connection has the peers node address - public boolean isConfirmed(NodeAddress nodeAddress) { - return networkNode.getNodeAddressesOfConfirmedConnections().contains(nodeAddress); - } - - public void handleConnectionFault(Connection connection) { - connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> handleConnectionFault(nodeAddress, connection)); - } - - public void handleConnectionFault(NodeAddress nodeAddress) { - handleConnectionFault(nodeAddress, null); + public int getMaxConnections() { + return maxConnectionsAbsolute; } - public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection connection) { - log.debug("handleConnectionFault called: nodeAddress=" + nodeAddress); - boolean doRemovePersistedPeer = false; - removeReportedPeer(nodeAddress); - Optional persistedPeerOptional = getPersistedPeerOptional(nodeAddress); - if (persistedPeerOptional.isPresent()) { - Peer persistedPeer = persistedPeerOptional.get(); - persistedPeer.increaseFailedConnectionAttempts(); - doRemovePersistedPeer = persistedPeer.tooManyFailedConnectionAttempts(); - } - doRemovePersistedPeer = doRemovePersistedPeer || (connection != null && connection.getRuleViolation() != null); - if (doRemovePersistedPeer) - removePersistedPeer(nodeAddress); - else - removeTooOldPersistedPeers(); - } + /////////////////////////////////////////////////////////////////////////////////////////// + // Listeners + /////////////////////////////////////////////////////////////////////////////////////////// - public void shutDownConnection(Connection connection, CloseConnectionReason closeConnectionReason) { - if (connection.getPeerType() != Connection.PeerType.DIRECT_MSG_PEER) - connection.shutDown(closeConnectionReason); + public void addListener(Listener listener) { + listeners.add(listener); } - public void shutDownConnection(NodeAddress peersNodeAddress, CloseConnectionReason closeConnectionReason) { - networkNode.getAllConnections().stream() - .filter(connection -> connection.getPeersNodeAddressOptional().isPresent() && - connection.getPeersNodeAddressOptional().get().equals(peersNodeAddress) && - connection.getPeerType() != Connection.PeerType.DIRECT_MSG_PEER) - .findAny() - .ifPresent(connection -> connection.shutDown(closeConnectionReason)); + public void removeListener(Listener listener) { + listeners.remove(listener); } - // Delivers the live peers from the last 30 min (MAX_AGE_LIVE_PEERS) - // We include older peers to avoid risks for network partitioning - public Set getLivePeers(NodeAddress excludedNodeAddress) { - int oldNumLatestLivePeers = latestLivePeers.size(); - Set currentLivePeers = new HashSet<>(getConnectedReportedPeers().stream() - .filter(e -> !isSeedNode(e)) - .filter(e -> !e.getNodeAddress().equals(excludedNodeAddress)) - .collect(Collectors.toSet())); - latestLivePeers.addAll(currentLivePeers); - long maxAge = new Date().getTime() - MAX_AGE_LIVE_PEERS; - latestLivePeers = latestLivePeers.stream() - .filter(peer -> peer.getDate().getTime() > maxAge) - .collect(Collectors.toSet()); - if (oldNumLatestLivePeers != latestLivePeers.size()) - log.info("Num of latestLivePeers={}", latestLivePeers.size()); - return latestLivePeers; - } /////////////////////////////////////////////////////////////////////////////////////////// - // Private + // Private misc /////////////////////////////////////////////////////////////////////////////////////////// + // Modify this to change the relationships between connection limits. + // maxConnections default 12 + private void setConnectionLimits(int maxConnections) { + this.maxConnections = maxConnections; // app node 12; seedNode 30 + minConnections = Math.max(1, (int) Math.round(maxConnections * 0.7)); // app node 1-8; seedNode 21 + disconnectFromSeedNode = maxConnections; // app node 12; seedNode 30 + maxConnectionsPeer = Math.max(4, (int) Math.round(maxConnections * 1.3)); // app node 16; seedNode 39 + maxConnectionsNonDirect = Math.max(8, (int) Math.round(maxConnections * 1.7)); // app node 20; seedNode 51 + maxConnectionsAbsolute = Math.max(12, (int) Math.round(maxConnections * 2.5)); // app node 30; seedNode 66 + } + private Set getConnectedReportedPeers() { // networkNode.getConfirmedConnections includes: // filter(connection -> connection.getPeersNodeAddressOptional().isPresent()) @@ -682,17 +743,31 @@ private Set getConnectedReportedPeers() { // If we have a new connection the supportedCapabilities is empty. // We lookup if we have already stored the supportedCapabilities at the persisted or reported peers // and if so we use that. - if (supportedCapabilities.isEmpty()) { - Set allPeers = new HashSet<>(getPersistedPeers()); - allPeers.addAll(getReportedPeers()); - Optional ourPeer = allPeers.stream().filter(peer -> peer.getNodeAddress().equals(connection.getPeersNodeAddressOptional().get())) + Optional peersNodeAddressOptional = connection.getPeersNodeAddressOptional(); + checkArgument(peersNodeAddressOptional.isPresent()); // getConfirmedConnections delivers only connections where we know the address + NodeAddress peersNodeAddress = peersNodeAddressOptional.get(); + boolean capabilitiesNotFoundInConnection = supportedCapabilities.isEmpty(); + if (capabilitiesNotFoundInConnection) { + // If not found in connection we look up if we got the Capabilities set from any of the + // reported or persisted peers + Set persistedAndReported = new HashSet<>(getPersistedPeers()); + persistedAndReported.addAll(getReportedPeers()); + Optional candidate = persistedAndReported.stream() + .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) .filter(peer -> !peer.getCapabilities().isEmpty()) .findAny(); - if (ourPeer.isPresent()) - supportedCapabilities = new Capabilities(ourPeer.get().getCapabilities()); + if (candidate.isPresent()) { + supportedCapabilities = new Capabilities(candidate.get().getCapabilities()); + } + } + Peer peer = new Peer(peersNodeAddress, supportedCapabilities); + + // If we did not found the capability from our own connection we add a listener, + // so once we get a connection with that peer and exchange a message containing the capabilities + // we get set the capabilities. + if (capabilitiesNotFoundInConnection) { + connection.addWeakCapabilitiesListener(peer); } - Peer peer = new Peer(connection.getPeersNodeAddressOptional().get(), supportedCapabilities); - connection.addWeakCapabilitiesListener(peer); return peer; }) .collect(Collectors.toSet()); @@ -709,8 +784,8 @@ private void printConnectedPeers() { if (!networkNode.getConfirmedConnections().isEmpty()) { StringBuilder result = new StringBuilder("\n\n------------------------------------------------------------\n" + "Connected peers for node " + networkNode.getNodeAddress() + ":"); - networkNode.getConfirmedConnections().stream().forEach(e -> result.append("\n") - .append(e.getPeersNodeAddressOptional().get()).append(" ").append(e.getPeerType())); + networkNode.getConfirmedConnections().forEach(e -> result.append("\n") + .append(e.getPeersNodeAddressOptional()).append(" ").append(e.getPeerType())); result.append("\n------------------------------------------------------------\n"); log.debug(result.toString()); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java index 48433f3e10d..38d2bd3ca4b 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java @@ -210,8 +210,8 @@ public void onConnection(Connection connection) { public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { closeHandler(connection); - if (peerManager.isNodeBanned(closeConnectionReason, connection) && connection.getPeersNodeAddressOptional().isPresent()) { - final NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); + if (peerManager.isPeerBanned(closeConnectionReason, connection) && connection.getPeersNodeAddressOptional().isPresent()) { + NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); seedNodeAddresses.remove(nodeAddress); handlerMap.remove(nodeAddress); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java index 2297758ee30..2b11b0c64a9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java @@ -32,6 +32,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import java.util.HashSet; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -83,11 +84,11 @@ public GetPeersRequestHandler(NetworkNode networkNode, PeerManager peerManager, // API /////////////////////////////////////////////////////////////////////////////////////////// - public void handle(GetPeersRequest getPeersRequest, final Connection connection) { + public void handle(GetPeersRequest getPeersRequest, Connection connection) { checkArgument(connection.getPeersNodeAddressOptional().isPresent(), "The peers address must have been already set at the moment"); GetPeersResponse getPeersResponse = new GetPeersResponse(getPeersRequest.getNonce(), - peerManager.getLivePeers(connection.getPeersNodeAddressOptional().get())); + new HashSet<>(peerManager.getLivePeers(connection.getPeersNodeAddressOptional().get()))); checkArgument(timeoutTimer == null, "onGetPeersRequest must not be called twice."); timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions @@ -130,8 +131,9 @@ public void onFailure(@NotNull Throwable throwable) { } } }, MoreExecutors.directExecutor()); - - peerManager.addToReportedPeers(getPeersRequest.getReportedPeers(), connection); + peerManager.addToReportedPeers(getPeersRequest.getReportedPeers(), + connection, + getPeersRequest.getSupportedCapabilities()); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java index cb3f8d64e7e..99678ad6053 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java @@ -27,7 +27,6 @@ import java.util.Date; -import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -35,17 +34,15 @@ import javax.annotation.Nullable; @Getter -@EqualsAndHashCode(exclude = {"date"}) // failedConnectionAttempts is transient and therefore excluded anyway @Slf4j public final class Peer implements HasCapabilities, NetworkPayload, PersistablePayload, SupportedCapabilitiesListener { private static final int MAX_FAILED_CONNECTION_ATTEMPTS = 5; private final NodeAddress nodeAddress; private final long date; - // Added in v. 0.7.1 - @Setter transient private int failedConnectionAttempts = 0; + @Setter private Capabilities capabilities = new Capabilities(); public Peer(NodeAddress nodeAddress, @Nullable Capabilities supportedCapabilities) { @@ -83,10 +80,14 @@ public static Peer fromProto(protobuf.Peer proto) { // API /////////////////////////////////////////////////////////////////////////////////////////// - public void increaseFailedConnectionAttempts() { + public void onDisconnect() { this.failedConnectionAttempts++; } + public void onConnection() { + this.failedConnectionAttempts--; + } + public boolean tooManyFailedConnectionAttempts() { return failedConnectionAttempts >= MAX_FAILED_CONNECTION_ATTEMPTS; } @@ -95,6 +96,10 @@ public Date getDate() { return new Date(date); } + public long getDateAsLong() { + return date; + } + @Override public void onChanged(Capabilities supportedCapabilities) { if (!supportedCapabilities.isEmpty()) { @@ -102,14 +107,29 @@ public void onChanged(Capabilities supportedCapabilities) { } } + // We use only node address for equals and hashcode + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Peer)) return false; + + Peer peer = (Peer) o; + + return nodeAddress != null ? nodeAddress.equals(peer.nodeAddress) : peer.nodeAddress == null; + } + + @Override + public int hashCode() { + return nodeAddress != null ? nodeAddress.hashCode() : 0; + } @Override public String toString() { return "Peer{" + "\n nodeAddress=" + nodeAddress + - ",\n supportedCapabilities=" + capabilities + - ",\n failedConnectionAttempts=" + failedConnectionAttempts + ",\n date=" + date + + ",\n failedConnectionAttempts=" + failedConnectionAttempts + + ",\n capabilities=" + capabilities + "\n}"; } } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java index d0266c2ea36..384f05c34c5 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java @@ -35,6 +35,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import java.util.HashSet; import java.util.Random; import java.util.concurrent.TimeUnit; @@ -104,7 +105,9 @@ private void sendGetPeersRequest(NodeAddress nodeAddress) { log.debug("sendGetPeersRequest to nodeAddress={}", nodeAddress); if (!stopped) { if (networkNode.getNodeAddress() != null) { - GetPeersRequest getPeersRequest = new GetPeersRequest(networkNode.getNodeAddress(), nonce, peerManager.getLivePeers(nodeAddress)); + GetPeersRequest getPeersRequest = new GetPeersRequest(networkNode.getNodeAddress(), + nonce, + new HashSet<>(peerManager.getLivePeers(nodeAddress))); if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions if (!stopped) { @@ -168,7 +171,9 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // Check if the response is for our request if (getPeersResponse.getRequestNonce() == nonce) { - peerManager.addToReportedPeers(getPeersResponse.getReportedPeers(), connection); + peerManager.addToReportedPeers(getPeersResponse.getReportedPeers(), + connection, + getPeersResponse.getSupportedCapabilities()); cleanup(); listener.onComplete(); } else { diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java index ad8489a1790..a843abf92d8 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java @@ -147,8 +147,9 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection }, RETRY_DELAY_SEC); } - if (peerManager.isNodeBanned(closeConnectionReason, connection)) - seedNodeAddresses.remove(connection.getPeersNodeAddressOptional().get()); + if (peerManager.isPeerBanned(closeConnectionReason, connection)) { + connection.getPeersNodeAddressOptional().ifPresent(seedNodeAddresses::remove); + } } @Override @@ -224,7 +225,8 @@ public void onFault(String errorMessage, Connection connection) { /////////////////////////////////////////////////////////////////////////////////////////// private void requestReportedPeers(NodeAddress nodeAddress, List remainingNodeAddresses) { - log.debug("requestReportedPeers nodeAddress={}; remainingNodeAddresses.size={}", nodeAddress, remainingNodeAddresses.size()); + log.debug("requestReportedPeers nodeAddress={}; remainingNodeAddresses.size={}", + nodeAddress, remainingNodeAddresses.size()); if (!stopped) { if (!handlerMap.containsKey(nodeAddress)) { PeerExchangeHandler peerExchangeHandler = new PeerExchangeHandler(networkNode, diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java index d32a4952c03..37112ec96ea 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java @@ -21,46 +21,55 @@ import com.google.protobuf.Message; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +@Slf4j @EqualsAndHashCode public class PeerList implements PersistableEnvelope { @Getter - private final List list = new ArrayList<>(); + private final Set set = new HashSet<>(); public PeerList() { } - public PeerList(List list) { - setAll(list); + public PeerList(Set set) { + setAll(set); } public int size() { - return list.size(); + return set.size(); } @Override public Message toProtoMessage() { return protobuf.PersistableEnvelope.newBuilder() .setPeerList(protobuf.PeerList.newBuilder() - .addAllPeer(list.stream().map(Peer::toProtoMessage).collect(Collectors.toList()))) + .addAllPeer(set.stream().map(Peer::toProtoMessage).collect(Collectors.toList()))) .build(); } public static PeerList fromProto(protobuf.PeerList proto) { - return new PeerList(new ArrayList<>(proto.getPeerList().stream() + return new PeerList(proto.getPeerList().stream() .map(Peer::fromProto) - .collect(Collectors.toList()))); + .collect(Collectors.toSet())); } public void setAll(Collection collection) { - this.list.clear(); - this.list.addAll(collection); + this.set.clear(); + this.set.addAll(collection); + } + + @Override + public String toString() { + return "PeerList{" + + "\n set=" + set + + "\n}"; } } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java index 2f23ef4308b..90f31b97dae 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java @@ -47,8 +47,14 @@ public final class GetPeersRequest extends NetworkEnvelope implements PeerExchan @Nullable private final Capabilities supportedCapabilities; - public GetPeersRequest(NodeAddress senderNodeAddress, int nonce, Set reportedPeers) { - this(senderNodeAddress, nonce, reportedPeers, Capabilities.app, Version.getP2PMessageVersion()); + public GetPeersRequest(NodeAddress senderNodeAddress, + int nonce, + Set reportedPeers) { + this(senderNodeAddress, + nonce, + reportedPeers, + Capabilities.app, + Version.getP2PMessageVersion()); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java index 5a41fe91f40..149bcb4946e 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java @@ -43,8 +43,12 @@ public final class GetPeersResponse extends NetworkEnvelope implements PeerExcha @Nullable private final Capabilities supportedCapabilities; - public GetPeersResponse(int requestNonce, Set reportedPeers) { - this(requestNonce, reportedPeers, Capabilities.app, Version.getP2PMessageVersion()); + public GetPeersResponse(int requestNonce, + Set reportedPeers) { + this(requestNonce, + reportedPeers, + Capabilities.app, + Version.getP2PMessageVersion()); } diff --git a/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java b/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java index ce483889703..b225b25f553 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java @@ -24,6 +24,7 @@ public interface HashMapChangedListener { void onAdded(Collection protectedStorageEntries); - @SuppressWarnings("UnusedParameters") - void onRemoved(Collection protectedStorageEntries); + default void onRemoved(Collection protectedStorageEntries) { + // Often we are only interested in added data as there is no use case for remove + } } diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index f412e5146a4..7b9e3714cea 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -637,12 +637,8 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE // To avoid that expired data get stored and broadcast we check early for expire date. if (protectedStorageEntry.isExpired(clock)) { String peer = sender != null ? sender.getFullAddress() : "sender is null"; - log.warn("We received an expired protectedStorageEntry from peer {}. ProtectedStoragePayload={}", + log.debug("We received an expired protectedStorageEntry from peer {}. ProtectedStoragePayload={}", peer, protectedStorageEntry.getProtectedStoragePayload().getClass().getSimpleName()); - log.debug("Expired protectedStorageEntry from peer {}. getCreationTimeStamp={}, protectedStorageEntry={}", - peer, - new Date(protectedStorageEntry.getCreationTimeStamp()), - protectedStorageEntry); return false; } diff --git a/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java b/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java index 52f37bf44e5..7be3e4d1906 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java @@ -61,7 +61,7 @@ void put(P2PDataStorage.ByteArray hash, R payload) { requestPersistence(); } - R putIfAbsent(P2PDataStorage.ByteArray hash, R payload) { + protected R putIfAbsent(P2PDataStorage.ByteArray hash, R payload) { R previous = getMap().putIfAbsent(hash, payload); requestPersistence(); return previous; diff --git a/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java b/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java index adc1688593e..5d1b57d9120 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java @@ -117,12 +117,11 @@ protected T getStore(String fileName) { T persisted = persistenceManager.getPersisted(fileName); if (persisted != null) { store = persisted; - - int length = store.toProtoMessage().toByteArray().length; + /* int length = store.toProtoMessage().getSerializedSize(); double size = length > 1_000_000D ? length / 1_000_000D : length / 1_000D; String unit = length > 1_000_000D ? "MB" : "KB"; log.info("{}: size of {}: {} {}", this.getClass().getSimpleName(), - persisted.getClass().getSimpleName(), size, unit); + persisted.getClass().getSimpleName(), size, unit);*/ } else { store = createStore(); } diff --git a/p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET b/p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET deleted file mode 100644 index b43cfb50f36..00000000000 --- a/p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c6b7362593537425e1228ee9a7a234811dc23ec1f66db31be4cc896124ff20fa -size 15989339 diff --git a/p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET b/p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET new file mode 100644 index 00000000000..97de382b59c --- /dev/null +++ b/p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d58ed2b6b59006a13c31d32838959c3bf910430c6b88e357c967087e2f33a5c +size 3900357 diff --git a/p2p/src/test/java/bisq/network/p2p/MockNode.java b/p2p/src/test/java/bisq/network/p2p/MockNode.java index 9520cae156a..2112bcc39b9 100644 --- a/p2p/src/test/java/bisq/network/p2p/MockNode.java +++ b/p2p/src/test/java/bisq/network/p2p/MockNode.java @@ -60,7 +60,7 @@ public MockNode(int maxConnections) throws IOException { networkNode = mock(NetworkNode.class); File storageDir = Files.createTempDirectory("storage").toFile(); PersistenceManager persistenceManager = new PersistenceManager<>(storageDir, mock(PersistenceProtoResolver.class), mock(CorruptedStorageFileHandler.class)); - peerManager = new PeerManager(networkNode, mock(SeedNodeRepository.class), new ClockWatcher(), maxConnections, persistenceManager); + peerManager = new PeerManager(networkNode, mock(SeedNodeRepository.class), new ClockWatcher(), persistenceManager, maxConnections); connections = new HashSet<>(); when(networkNode.getAllConnections()).thenReturn(connections); } diff --git a/proto/src/main/proto/grpc.proto b/proto/src/main/proto/grpc.proto index 49f47e156ec..2d23a43b2d3 100644 --- a/proto/src/main/proto/grpc.proto +++ b/proto/src/main/proto/grpc.proto @@ -154,7 +154,7 @@ message GetTradeStatisticsRequest { } message GetTradeStatisticsReply { - repeated TradeStatistics2 TradeStatistics = 1; + repeated TradeStatistics3 TradeStatistics = 1; } /////////////////////////////////////////////////////////////////////////////////////////// diff --git a/proto/src/main/proto/pb.proto b/proto/src/main/proto/pb.proto index 11c32109811..8d7b152bd27 100644 --- a/proto/src/main/proto/pb.proto +++ b/proto/src/main/proto/pb.proto @@ -510,9 +510,7 @@ message StoragePayload { Mediator mediator = 3; Filter filter = 4; - // not used anymore from v0.6 on. But leave it for receiving TradeStatistics objects from older - // versions and convert it to TradeStatistics2 objects. - TradeStatistics trade_statistics = 5 [deprecated = true]; + // TradeStatistics trade_statistics = 5 [deprecated = true]; Removed in v.1.4.0 MailboxStoragePayload mailbox_storage_payload = 6; OfferPayload offer_payload = 7; @@ -524,10 +522,11 @@ message StoragePayload { message PersistableNetworkPayload { oneof message { AccountAgeWitness account_age_witness = 1; - TradeStatistics2 trade_statistics2 = 2; + TradeStatistics2 trade_statistics2 = 2 [deprecated = true]; ProposalPayload proposal_payload = 3; BlindVotePayload blind_vote_payload = 4; SignedWitness signed_witness = 5; + TradeStatistics3 trade_statistics3 = 6; } } @@ -650,44 +649,36 @@ message Filter { bool disable_auto_conf = 24; } -// not used anymore from v0.6 on. But leave it for receiving TradeStatistics objects from older -// versions and convert it to TradeStatistics2 objects. -message TradeStatistics { - string base_currency = 1; - string counter_currency = 2; - OfferPayload.Direction direction = 3; - int64 trade_price = 4; - int64 trade_amount = 5; - int64 trade_date = 6; - string payment_method_id = 7; - int64 offer_date = 8; - bool offer_use_market_based_price = 9; - double offer_market_price_margin = 10; - int64 offer_amount = 11; - int64 offer_min_amount = 12; - string offer_id = 13; - string deposit_tx_id = 14; - bytes signature_pub_key_bytes = 15; - map extra_data = 16; -} - +// Deprecated message TradeStatistics2 { - string base_currency = 1; - string counter_currency = 2; - OfferPayload.Direction direction = 3; - int64 trade_price = 4; - int64 trade_amount = 5; - int64 trade_date = 6; - string payment_method_id = 7; - int64 offer_date = 8; - bool offer_use_market_based_price = 9; - double offer_market_price_margin = 10; - int64 offer_amount = 11; - int64 offer_min_amount = 12; - string offer_id = 13; - string deposit_tx_id = 14; - bytes hash = 15; - map extra_data = 16; + string base_currency = 1 [deprecated = true]; + string counter_currency = 2 [deprecated = true]; + OfferPayload.Direction direction = 3 [deprecated = true]; + int64 trade_price = 4 [deprecated = true]; + int64 trade_amount = 5 [deprecated = true]; + int64 trade_date = 6 [deprecated = true]; + string payment_method_id = 7 [deprecated = true]; + int64 offer_date = 8 [deprecated = true]; + bool offer_use_market_based_price = 9 [deprecated = true]; + double offer_market_price_margin = 10 [deprecated = true]; + int64 offer_amount = 11 [deprecated = true]; + int64 offer_min_amount = 12 [deprecated = true]; + string offer_id = 13 [deprecated = true]; + string deposit_tx_id = 14 [deprecated = true]; + bytes hash = 15 [deprecated = true]; + map extra_data = 16 [deprecated = true]; +} + +message TradeStatistics3 { + string currency = 1; + int64 price = 2; + int64 amount = 3; + string payment_method = 4; + int64 date = 5; + string mediator = 6; + string refund_agent = 7; + bytes hash = 8; + map extra_data = 9; } message MailboxStoragePayload { @@ -1152,7 +1143,7 @@ message PersistableEnvelope { // BsqState bsq_state = 12; // not used but as other non-dao data have a higher index number we leave it to make clear that we cannot change following indexes AccountAgeWitnessStore account_age_witness_store = 13; - TradeStatistics2Store trade_statistics2_store = 14; + TradeStatistics2Store trade_statistics2_store = 14 [deprecated = true]; // PersistableNetworkPayloadList persistable_network_payload_list = 15; // long deprecated & migration away from it is already done @@ -1171,6 +1162,7 @@ message PersistableEnvelope { SignedWitnessStore signed_witness_store = 28; MediationDisputeList mediation_dispute_list = 29; RefundDisputeList refund_dispute_list = 30; + TradeStatistics3Store trade_statistics3_store = 31; } } @@ -1211,8 +1203,13 @@ message SignedWitnessStore { } // We use a list not a hash map to save disc space. The hash can be calculated from the payload anyway +// Deprecated message TradeStatistics2Store { - repeated TradeStatistics2 items = 1; + repeated TradeStatistics2 items = 1 [deprecated = true]; +} + +message TradeStatistics3Store { + repeated TradeStatistics3 items = 1; } message PeerList {