New trade statistics

common/src/main/java/bisq/common/app/Capabilities.java

@@ -98,6 +98,10 @@ public boolean containsAll(Capability... capabilities) {
         return this.capabilities.containsAll(Arrays.asList(capabilities));
     }
 
+    public boolean contains(Capability capability) {
+        return this.capabilities.contains(capability);
+    }
+
     public boolean isEmpty() {
         return capabilities.isEmpty();
     }
@@ -180,4 +184,18 @@ public String prettyPrint() {
     public int size() {
         return capabilities.size();
     }
+
+    // We return true if our capabilities have less capabilities than the parameter value
+    public boolean hasLess(Capabilities other) {
+        return findHighestCapability(this) < findHighestCapability(other);
+    }
+
+    // We use the sum of all capabilities. Alternatively we could use the highest entry.
+    // Neither would support removal of past capabilities, a use case we never had so far and which might have
+    // backward compatibility issues, so we should treat capabilities as an append-only data structure.
+    public int findHighestCapability(Capabilities capabilities) {
+        return (int) capabilities.capabilities.stream()
+                .mapToLong(e -> (long) e.ordinal())
+                .sum();
+    }
 }

common/src/main/java/bisq/common/app/Capability.java

@@ -40,5 +40,7 @@ public enum Capability {
     SIGNED_ACCOUNT_AGE_WITNESS,         // Supports the signed account age witness feature
     MEDIATION,                          // Supports mediation feature
     REFUND_AGENT,                       // Supports refund agents
-    TRADE_STATISTICS_HASH_UPDATE        // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data.
+    TRADE_STATISTICS_HASH_UPDATE,       // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data.
+    NO_ADDRESS_PRE_FIX,                 // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix.
+    TRADE_STATISTICS_3                  // We used a new reduced trade statistics model from v1.4.0 on
 }

common/src/main/java/bisq/common/app/Version.java

@@ -128,7 +128,6 @@ public static void printVersion() {
                 '}');
     }
 
-    //TODO move to consensus area
     public static final byte COMPENSATION_REQUEST = (byte) 0x01;
     public static final byte REIMBURSEMENT_REQUEST = (byte) 0x01;
     public static final byte PROPOSAL = (byte) 0x01;

common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java

@@ -24,8 +24,5 @@
  * Better to use the excludeFromJsonDataMap (annotated with @JsonExclude; used in PaymentAccountPayload) to
  * add a key/value pair.
  */
-// TODO PubKeyRing and NodeAddress (network) are using UsedForTradeContractJson that is why it is in common module,
-// which is a bit weird... Maybe we need either rename common or split it to util and common where common is common code
-// used in network and core?
 public interface UsedForTradeContractJson {
 }

common/src/main/java/bisq/common/crypto/Encryption.java

@@ -48,7 +48,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-// TODO is Hmac needed/make sense?
 public class Encryption {
     private static final Logger log = LoggerFactory.getLogger(Encryption.class);
 

common/src/main/java/bisq/common/crypto/KeyStorage.java

@@ -55,7 +55,6 @@
 
 import static bisq.common.util.Preconditions.checkDir;
 
-// TODO: use a password protection for key storage
 @Singleton
 public class KeyStorage {
     private static final Logger log = LoggerFactory.getLogger(KeyStorage.class);

common/src/main/java/bisq/common/file/JsonFileManager.java

@@ -30,7 +30,7 @@
 
 @Slf4j
 public class JsonFileManager {
-    private final ThreadPoolExecutor executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60);
+    private final ThreadPoolExecutor executor;
     private final File dir;
 
 
@@ -41,6 +41,8 @@ public class JsonFileManager {
     public JsonFileManager(File dir) {
         this.dir = dir;
 
+        this.executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60);
+
         if (!dir.exists())
             if (!dir.mkdir())
                 log.warn("make dir failed");

common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java

@@ -48,7 +48,6 @@ public Message toProtoMessage() {
         return getNetworkEnvelopeBuilder().build();
     }
 
-    // todo remove
     public protobuf.NetworkEnvelope toProtoNetworkEnvelope() {
         return getNetworkEnvelopeBuilder().build();
     }

common/src/test/java/bisq/common/app/CapabilitiesTest.java

@@ -23,6 +23,7 @@
 
 import org.junit.Test;
 
+import static bisq.common.app.Capability.DAO_FULL_NODE;
 import static bisq.common.app.Capability.SEED_NODE;
 import static bisq.common.app.Capability.TRADE_STATISTICS;
 import static bisq.common.app.Capability.TRADE_STATISTICS_2;
@@ -40,6 +41,47 @@ public void testNoCapabilitiesAvailable() {
         assertFalse(DUT.containsAll(new Capabilities(SEED_NODE)));
     }
 
+    @Test
+    public void testHasLess() {
+        assertTrue(new Capabilities().hasLess(new Capabilities(SEED_NODE)));
+        assertFalse(new Capabilities().hasLess(new Capabilities()));
+        assertFalse(new Capabilities(SEED_NODE).hasLess(new Capabilities()));
+        assertTrue(new Capabilities(SEED_NODE).hasLess(new Capabilities(DAO_FULL_NODE)));
+        assertFalse(new Capabilities(DAO_FULL_NODE).hasLess(new Capabilities(SEED_NODE)));
+
+        Capabilities all = new Capabilities(
+                TRADE_STATISTICS,
+                TRADE_STATISTICS_2,
+                Capability.ACCOUNT_AGE_WITNESS,
+                Capability.ACK_MSG,
+                Capability.PROPOSAL,
+                Capability.BLIND_VOTE,
+                Capability.DAO_STATE,
+                Capability.BUNDLE_OF_ENVELOPES,
+                Capability.MEDIATION,
+                Capability.SIGNED_ACCOUNT_AGE_WITNESS,
+                Capability.REFUND_AGENT,
+                Capability.TRADE_STATISTICS_HASH_UPDATE
+        );
+        Capabilities other = new Capabilities(
+                TRADE_STATISTICS,
+                TRADE_STATISTICS_2,
+                Capability.ACCOUNT_AGE_WITNESS,
+                Capability.ACK_MSG,
+                Capability.PROPOSAL,
+                Capability.BLIND_VOTE,
+                Capability.DAO_STATE,
+                Capability.BUNDLE_OF_ENVELOPES,
+                Capability.MEDIATION,
+                Capability.SIGNED_ACCOUNT_AGE_WITNESS,
+                Capability.REFUND_AGENT,
+                Capability.TRADE_STATISTICS_HASH_UPDATE,
+                Capability.NO_ADDRESS_PRE_FIX
+        );
+
+        assertTrue(all.hasLess(other));
+    }
+
     @Test
     public void testO() {
         Capabilities DUT = new Capabilities(TRADE_STATISTICS);

core/src/main/java/bisq/core/api/CoreApi.java

@@ -22,7 +22,7 @@
 import bisq.core.offer.Offer;
 import bisq.core.offer.OfferPayload;
 import bisq.core.payment.PaymentAccount;
-import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 import bisq.core.trade.statistics.TradeStatisticsManager;
 
 import bisq.common.app.Version;
@@ -196,7 +196,7 @@ public void removeWalletPassword(String password) {
         walletsService.removeWalletPassword(password);
     }
 
-    public List<TradeStatistics2> getTradeStatistics() {
+    public List<TradeStatistics3> getTradeStatistics() {
         return new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet());
     }
 

core/src/main/java/bisq/core/app/BisqExecutable.java

@@ -25,6 +25,7 @@
 import bisq.core.setup.CorePersistedDataHost;
 import bisq.core.setup.CoreSetup;
 import bisq.core.support.dispute.arbitration.arbitrator.ArbitratorManager;
+import bisq.core.trade.statistics.TradeStatisticsManager;
 import bisq.core.trade.txproof.xmr.XmrTxProofService;
 
 import bisq.network.p2p.P2PService;
@@ -220,6 +221,7 @@ public void gracefulShutDown(ResultHandler resultHandler) {
 
         try {
             injector.getInstance(ArbitratorManager.class).shutDown();
+            injector.getInstance(TradeStatisticsManager.class).shutDown();
             injector.getInstance(XmrTxProofService.class).shutDown();
             injector.getInstance(DaoSetup.class).shutDown();
             injector.getInstance(AvoidStandbyModeService.class).shutDown();

core/src/main/java/bisq/core/dao/governance/asset/AssetService.java

@@ -154,10 +154,10 @@ public void updateAssetStates() {
         // TradeAmountDateTuple object holding only the data we need.
         Map<String, List<TradeAmountDateTuple>> lookupMap = new HashMap<>();
         tradeStatisticsManager.getObservableTradeStatisticsSet().stream()
-                .filter(e -> CurrencyUtil.isCryptoCurrency(e.getBaseCurrency()))
+                .filter(e -> CurrencyUtil.isCryptoCurrency(e.getCurrency()))
                 .forEach(e -> {
-                    lookupMap.putIfAbsent(e.getBaseCurrency(), new ArrayList<>());
-                    lookupMap.get(e.getBaseCurrency()).add(new TradeAmountDateTuple(e.getTradeAmount().getValue(), e.getTradeDate().getTime()));
+                    lookupMap.putIfAbsent(e.getCurrency(), new ArrayList<>());
+                    lookupMap.get(e.getCurrency()).add(new TradeAmountDateTuple(e.getAmount(), e.getDate()));
                 });
 
         getStatefulAssets().stream()

core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java

@@ -170,7 +170,7 @@ private void connectToNextNode() {
     private void connectToAnyFullNode() {
         Capabilities required = new Capabilities(Capability.DAO_FULL_NODE);
 
-        List<Peer> list = peerManager.getLivePeers(null).stream()
+        List<Peer> list = peerManager.getLivePeers().stream()
                 .filter(peer -> peer.getCapabilities().containsAll(required))
                 .collect(Collectors.toList());
 

core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java

@@ -180,7 +180,7 @@ public void onConnection(Connection connection) {
     public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) {
         closeHandler(connection);
 
-        if (peerManager.isNodeBanned(closeConnectionReason, connection)) {
+        if (peerManager.isPeerBanned(closeConnectionReason, connection)) {
             connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> {
                 seedNodeAddresses.remove(nodeAddress);
                 removeFromRequestBlocksHandlerMap(nodeAddress);

core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java

@@ -19,7 +19,7 @@
 
 import bisq.core.support.dispute.agent.DisputeAgent;
 import bisq.core.support.dispute.agent.DisputeAgentManager;
-import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 import bisq.core.trade.statistics.TradeStatisticsManager;
 
 import bisq.common.util.Tuple2;
@@ -42,36 +42,37 @@
 
 @Slf4j
 public class DisputeAgentSelection {
+    public static final int LOOK_BACK_RANGE = 100;
+
     public static <T extends DisputeAgent> T getLeastUsedMediator(TradeStatisticsManager tradeStatisticsManager,
                                                                   DisputeAgentManager<T> disputeAgentManager) {
         return getLeastUsedDisputeAgent(tradeStatisticsManager,
                 disputeAgentManager,
-                TradeStatistics2.MEDIATOR_ADDRESS);
+                true);
     }
 
     public static <T extends DisputeAgent> T getLeastUsedRefundAgent(TradeStatisticsManager tradeStatisticsManager,
                                                                      DisputeAgentManager<T> disputeAgentManager) {
         return getLeastUsedDisputeAgent(tradeStatisticsManager,
                 disputeAgentManager,
-                TradeStatistics2.REFUND_AGENT_ADDRESS);
+                false);
     }
 
     private static <T extends DisputeAgent> T getLeastUsedDisputeAgent(TradeStatisticsManager tradeStatisticsManager,
                                                                        DisputeAgentManager<T> disputeAgentManager,
-                                                                       String extraMapKey) {
+                                                                       boolean isMediator) {
         // We take last 100 entries from trade statistics
-        List<TradeStatistics2> list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet());
-        list.sort(Comparator.comparing(TradeStatistics2::getTradeDate));
+        List<TradeStatistics3> list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet());
+        list.sort(Comparator.comparing(TradeStatistics3::getDate));
         Collections.reverse(list);
         if (!list.isEmpty()) {
-            int max = Math.min(list.size(), 100);
+            int max = Math.min(list.size(), LOOK_BACK_RANGE);
             list = list.subList(0, max);
         }
 
         // We stored only first 4 chars of disputeAgents onion address
         List<String> lastAddressesUsedInTrades = list.stream()
-                .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap() != null)
-                .map(tradeStatistics2 -> tradeStatistics2.getExtraDataMap().get(extraMapKey))
+                .map(tradeStatistics3 -> isMediator ? tradeStatistics3.getMediator() : tradeStatistics3.getRefundAgent())
                 .filter(Objects::nonNull)
                 .collect(Collectors.toList());
 

core/src/main/java/bisq/core/proto/CoreProtoResolver.java

@@ -54,6 +54,7 @@
 import bisq.core.payment.payload.WeChatPayAccountPayload;
 import bisq.core.payment.payload.WesternUnionAccountPayload;
 import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 
 import bisq.common.proto.ProtoResolver;
 import bisq.common.proto.ProtobufferRuntimeException;
@@ -178,6 +179,8 @@ public PersistablePayload fromProto(protobuf.PersistableNetworkPayload proto) {
                     return BlindVotePayload.fromProto(proto.getBlindVotePayload());
                 case SIGNED_WITNESS:
                     return SignedWitness.fromProto(proto.getSignedWitness());
+                case TRADE_STATISTICS3:
+                    return TradeStatistics3.fromProto(proto.getTradeStatistics3());
                 default:
                     throw new ProtobufferRuntimeException("Unknown proto message case (PB.PersistableNetworkPayload). messageCase=" + proto.getMessageCase());
             }

core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java

@@ -59,7 +59,6 @@
 import bisq.core.trade.messages.PeerPublishedDelayedPayoutTxMessage;
 import bisq.core.trade.messages.RefreshTradeStateRequest;
 import bisq.core.trade.messages.TraderSignedWitnessMessage;
-import bisq.core.trade.statistics.TradeStatistics;
 
 import bisq.network.p2p.AckMessage;
 import bisq.network.p2p.BundleOfEnvelopes;
@@ -265,9 +264,6 @@ public NetworkPayload fromProto(protobuf.StoragePayload proto) {
                     return RefundAgent.fromProto(proto.getRefundAgent());
                 case FILTER:
                     return Filter.fromProto(proto.getFilter());
-                case TRADE_STATISTICS:
-                    // Still used to convert TradeStatistics data from pre v0.6 versions
-                    return TradeStatistics.fromProto(proto.getTradeStatistics());
                 case MAILBOX_STORAGE_PAYLOAD:
                     return MailboxStoragePayload.fromProto(proto.getMailboxStoragePayload());
                 case OFFER_PAYLOAD:

core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java

@@ -39,6 +39,7 @@
 import bisq.core.support.dispute.refund.RefundDisputeList;
 import bisq.core.trade.TradableList;
 import bisq.core.trade.statistics.TradeStatistics2Store;
+import bisq.core.trade.statistics.TradeStatistics3Store;
 import bisq.core.user.PreferencesPayload;
 import bisq.core.user.UserPayload;
 
@@ -126,6 +127,8 @@ public PersistableEnvelope fromProto(protobuf.PersistableEnvelope proto) {
                     return UnconfirmedBsqChangeOutputList.fromProto(proto.getUnconfirmedBsqChangeOutputList());
                 case SIGNED_WITNESS_STORE:
                     return SignedWitnessStore.fromProto(proto.getSignedWitnessStore());
+                case TRADE_STATISTICS3_STORE:
+                    return TradeStatistics3Store.fromProto(proto.getTradeStatistics3Store());
 
                 default:
                     throw new ProtobufferRuntimeException("Unknown proto message case(PB.PersistableEnvelope). " +

core/src/main/java/bisq/core/provider/fee/FeeService.java

@@ -52,7 +52,6 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-// TODO use dao parameters for fee
 @Slf4j
 public class FeeService {
 

core/src/main/java/bisq/core/provider/price/PriceFeedService.java

@@ -23,7 +23,7 @@
 import bisq.core.monetary.Price;
 import bisq.core.provider.PriceNodeHttpClient;
 import bisq.core.provider.ProvidersRepository;
-import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 import bisq.core.user.Preferences;
 
 import bisq.network.http.HttpClient;
@@ -50,6 +50,7 @@
 import java.time.Instant;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -288,12 +289,12 @@ public Date getLastRequestTimeStamp() {
         return new Date(epochInMillisAtLastRequest);
     }
 
-    public void applyLatestBisqMarketPrice(Set<TradeStatistics2> tradeStatisticsSet) {
+    public void applyLatestBisqMarketPrice(Set<TradeStatistics3> tradeStatisticsSet) {
         // takes about 10 ms for 5000 items
-        Map<String, List<TradeStatistics2>> mapByCurrencyCode = new HashMap<>();
+        Map<String, List<TradeStatistics3>> mapByCurrencyCode = new HashMap<>();
         tradeStatisticsSet.forEach(e -> {
-            final List<TradeStatistics2> list;
-            final String currencyCode = e.getCurrencyCode();
+            List<TradeStatistics3> list;
+            String currencyCode = e.getCurrency();
             if (mapByCurrencyCode.containsKey(currencyCode)) {
                 list = mapByCurrencyCode.get(currencyCode);
             } else {
@@ -306,9 +307,9 @@ public void applyLatestBisqMarketPrice(Set<TradeStatistics2> tradeStatisticsSet)
         mapByCurrencyCode.values().stream()
                 .filter(list -> !list.isEmpty())
                 .forEach(list -> {
-                    list.sort((o1, o2) -> o1.getTradeDate().compareTo(o2.getTradeDate()));
-                    TradeStatistics2 tradeStatistics = list.get(list.size() - 1);
-                    setBisqMarketPrice(tradeStatistics.getCurrencyCode(), tradeStatistics.getTradePrice());
+                    list.sort(Comparator.comparing(TradeStatistics3::getTradeDate));
+                    TradeStatistics3 tradeStatistics = list.get(list.size() - 1);
+                    setBisqMarketPrice(tradeStatistics.getCurrency(), tradeStatistics.getTradePrice());
                 });
     }
 

core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java

@@ -39,7 +39,9 @@ static void setSupportedCapabilities(Config config) {
                 Capability.MEDIATION,
                 Capability.SIGNED_ACCOUNT_AGE_WITNESS,
                 Capability.REFUND_AGENT,
-                Capability.TRADE_STATISTICS_HASH_UPDATE
+                Capability.TRADE_STATISTICS_HASH_UPDATE,
+                Capability.NO_ADDRESS_PRE_FIX,
+                Capability.TRADE_STATISTICS_3
         );
 
         if (config.daoActivated) {