New trade statistics

common/src/main/java/bisq/common/app/Capability.java

@@ -41,5 +41,6 @@ public enum Capability {
     MEDIATION,                          // Supports mediation feature
     REFUND_AGENT,                       // Supports refund agents
     TRADE_STATISTICS_HASH_UPDATE,       // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data.
-    NO_ADDRESS_PRE_FIX                  // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix.
+    NO_ADDRESS_PRE_FIX,                 // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix.
+    TRADE_STATISTICS_3                  // We used a new reduced trade statistics model from v1.4.0 on
 }

common/src/main/java/bisq/common/app/Version.java

@@ -128,7 +128,6 @@ public static void printVersion() {
                 '}');
     }
 
-    //TODO move to consensus area
     public static final byte COMPENSATION_REQUEST = (byte) 0x01;
     public static final byte REIMBURSEMENT_REQUEST = (byte) 0x01;
     public static final byte PROPOSAL = (byte) 0x01;

common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java

@@ -24,8 +24,5 @@
  * Better to use the excludeFromJsonDataMap (annotated with @JsonExclude; used in PaymentAccountPayload) to
  * add a key/value pair.
  */
-// TODO PubKeyRing and NodeAddress (network) are using UsedForTradeContractJson that is why it is in common module,
-// which is a bit weird... Maybe we need either rename common or split it to util and common where common is common code
-// used in network and core?
 public interface UsedForTradeContractJson {
 }

common/src/main/java/bisq/common/crypto/Encryption.java

@@ -48,7 +48,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-// TODO is Hmac needed/make sense?
 public class Encryption {
     private static final Logger log = LoggerFactory.getLogger(Encryption.class);
 

common/src/main/java/bisq/common/crypto/KeyStorage.java

@@ -55,7 +55,6 @@
 
 import static bisq.common.util.Preconditions.checkDir;
 
-// TODO: use a password protection for key storage
 @Singleton
 public class KeyStorage {
     private static final Logger log = LoggerFactory.getLogger(KeyStorage.class);

common/src/main/java/bisq/common/file/JsonFileManager.java

@@ -30,7 +30,7 @@
 
 @Slf4j
 public class JsonFileManager {
-    private final ThreadPoolExecutor executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60);
+    private final ThreadPoolExecutor executor;
     private final File dir;
 
 
@@ -41,6 +41,8 @@ public class JsonFileManager {
     public JsonFileManager(File dir) {
         this.dir = dir;
 
+        this.executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60);
+
         if (!dir.exists())
             if (!dir.mkdir())
                 log.warn("make dir failed");

common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java

@@ -48,7 +48,6 @@ public Message toProtoMessage() {
         return getNetworkEnvelopeBuilder().build();
     }
 
-    // todo remove
     public protobuf.NetworkEnvelope toProtoNetworkEnvelope() {
         return getNetworkEnvelopeBuilder().build();
     }

common/src/test/java/bisq/common/app/CapabilitiesTest.java

@@ -50,8 +50,8 @@ public void testHasLess() {
         assertFalse(new Capabilities(DAO_FULL_NODE).hasLess(new Capabilities(SEED_NODE)));
 
         Capabilities all = new Capabilities(
-                Capability.TRADE_STATISTICS,
-                Capability.TRADE_STATISTICS_2,
+                TRADE_STATISTICS,
+                TRADE_STATISTICS_2,
                 Capability.ACCOUNT_AGE_WITNESS,
                 Capability.ACK_MSG,
                 Capability.PROPOSAL,
@@ -64,8 +64,8 @@ public void testHasLess() {
                 Capability.TRADE_STATISTICS_HASH_UPDATE
         );
         Capabilities other = new Capabilities(
-                Capability.TRADE_STATISTICS,
-                Capability.TRADE_STATISTICS_2,
+                TRADE_STATISTICS,
+                TRADE_STATISTICS_2,
                 Capability.ACCOUNT_AGE_WITNESS,
                 Capability.ACK_MSG,
                 Capability.PROPOSAL,

core/src/main/java/bisq/core/api/CoreApi.java

@@ -22,7 +22,7 @@
 import bisq.core.offer.Offer;
 import bisq.core.offer.OfferPayload;
 import bisq.core.payment.PaymentAccount;
-import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 import bisq.core.trade.statistics.TradeStatisticsManager;
 
 import bisq.common.app.Version;
@@ -196,7 +196,7 @@ public void removeWalletPassword(String password) {
         walletsService.removeWalletPassword(password);
     }
 
-    public List<TradeStatistics2> getTradeStatistics() {
+    public List<TradeStatistics3> getTradeStatistics() {
         return new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet());
     }
 

core/src/main/java/bisq/core/app/BisqExecutable.java

@@ -25,6 +25,7 @@
 import bisq.core.setup.CorePersistedDataHost;
 import bisq.core.setup.CoreSetup;
 import bisq.core.support.dispute.arbitration.arbitrator.ArbitratorManager;
+import bisq.core.trade.statistics.TradeStatisticsManager;
 import bisq.core.trade.txproof.xmr.XmrTxProofService;
 
 import bisq.network.p2p.P2PService;
@@ -220,6 +221,7 @@ public void gracefulShutDown(ResultHandler resultHandler) {
 
         try {
             injector.getInstance(ArbitratorManager.class).shutDown();
+            injector.getInstance(TradeStatisticsManager.class).shutDown();
             injector.getInstance(XmrTxProofService.class).shutDown();
             injector.getInstance(DaoSetup.class).shutDown();
             injector.getInstance(AvoidStandbyModeService.class).shutDown();

core/src/main/java/bisq/core/dao/governance/asset/AssetService.java

@@ -154,10 +154,10 @@ public void updateAssetStates() {
         // TradeAmountDateTuple object holding only the data we need.
         Map<String, List<TradeAmountDateTuple>> lookupMap = new HashMap<>();
         tradeStatisticsManager.getObservableTradeStatisticsSet().stream()
-                .filter(e -> CurrencyUtil.isCryptoCurrency(e.getBaseCurrency()))
+                .filter(e -> CurrencyUtil.isCryptoCurrency(e.getCurrency()))
                 .forEach(e -> {
-                    lookupMap.putIfAbsent(e.getBaseCurrency(), new ArrayList<>());
-                    lookupMap.get(e.getBaseCurrency()).add(new TradeAmountDateTuple(e.getTradeAmount().getValue(), e.getTradeDate().getTime()));
+                    lookupMap.putIfAbsent(e.getCurrency(), new ArrayList<>());
+                    lookupMap.get(e.getCurrency()).add(new TradeAmountDateTuple(e.getAmount(), e.getDate()));
                 });
 
         getStatefulAssets().stream()

core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java

@@ -19,7 +19,7 @@
 
 import bisq.core.support.dispute.agent.DisputeAgent;
 import bisq.core.support.dispute.agent.DisputeAgentManager;
-import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 import bisq.core.trade.statistics.TradeStatisticsManager;
 
 import bisq.common.util.Tuple2;
@@ -42,36 +42,37 @@
 
 @Slf4j
 public class DisputeAgentSelection {
+    public static final int LOOK_BACK_RANGE = 100;
+
     public static <T extends DisputeAgent> T getLeastUsedMediator(TradeStatisticsManager tradeStatisticsManager,
                                                                   DisputeAgentManager<T> disputeAgentManager) {
         return getLeastUsedDisputeAgent(tradeStatisticsManager,
                 disputeAgentManager,
-                TradeStatistics2.MEDIATOR_ADDRESS);
+                true);
     }
 
     public static <T extends DisputeAgent> T getLeastUsedRefundAgent(TradeStatisticsManager tradeStatisticsManager,
                                                                      DisputeAgentManager<T> disputeAgentManager) {
         return getLeastUsedDisputeAgent(tradeStatisticsManager,
                 disputeAgentManager,
-                TradeStatistics2.REFUND_AGENT_ADDRESS);
+                false);
     }
 
     private static <T extends DisputeAgent> T getLeastUsedDisputeAgent(TradeStatisticsManager tradeStatisticsManager,
                                                                        DisputeAgentManager<T> disputeAgentManager,
-                                                                       String extraMapKey) {
+                                                                       boolean isMediator) {
         // We take last 100 entries from trade statistics
-        List<TradeStatistics2> list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet());
-        list.sort(Comparator.comparing(TradeStatistics2::getTradeDate));
+        List<TradeStatistics3> list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet());
+        list.sort(Comparator.comparing(TradeStatistics3::getDate));
         Collections.reverse(list);
         if (!list.isEmpty()) {
-            int max = Math.min(list.size(), 100);
+            int max = Math.min(list.size(), LOOK_BACK_RANGE);
             list = list.subList(0, max);
         }
 
         // We stored only first 4 chars of disputeAgents onion address
         List<String> lastAddressesUsedInTrades = list.stream()
-                .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap() != null)
-                .map(tradeStatistics2 -> tradeStatistics2.getExtraDataMap().get(extraMapKey))
+                .map(tradeStatistics3 -> isMediator ? tradeStatistics3.getMediator() : tradeStatistics3.getRefundAgent())
                 .filter(Objects::nonNull)
                 .collect(Collectors.toList());
 

core/src/main/java/bisq/core/proto/CoreProtoResolver.java

@@ -54,6 +54,7 @@
 import bisq.core.payment.payload.WeChatPayAccountPayload;
 import bisq.core.payment.payload.WesternUnionAccountPayload;
 import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 
 import bisq.common.proto.ProtoResolver;
 import bisq.common.proto.ProtobufferRuntimeException;
@@ -178,6 +179,8 @@ public PersistablePayload fromProto(protobuf.PersistableNetworkPayload proto) {
                     return BlindVotePayload.fromProto(proto.getBlindVotePayload());
                 case SIGNED_WITNESS:
                     return SignedWitness.fromProto(proto.getSignedWitness());
+                case TRADE_STATISTICS3:
+                    return TradeStatistics3.fromProto(proto.getTradeStatistics3());
                 default:
                     throw new ProtobufferRuntimeException("Unknown proto message case (PB.PersistableNetworkPayload). messageCase=" + proto.getMessageCase());
             }

core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java

@@ -59,7 +59,6 @@
 import bisq.core.trade.messages.PeerPublishedDelayedPayoutTxMessage;
 import bisq.core.trade.messages.RefreshTradeStateRequest;
 import bisq.core.trade.messages.TraderSignedWitnessMessage;
-import bisq.core.trade.statistics.TradeStatistics;
 
 import bisq.network.p2p.AckMessage;
 import bisq.network.p2p.BundleOfEnvelopes;
@@ -265,9 +264,6 @@ public NetworkPayload fromProto(protobuf.StoragePayload proto) {
                     return RefundAgent.fromProto(proto.getRefundAgent());
                 case FILTER:
                     return Filter.fromProto(proto.getFilter());
-                case TRADE_STATISTICS:
-                    // Still used to convert TradeStatistics data from pre v0.6 versions
-                    return TradeStatistics.fromProto(proto.getTradeStatistics());
                 case MAILBOX_STORAGE_PAYLOAD:
                     return MailboxStoragePayload.fromProto(proto.getMailboxStoragePayload());
                 case OFFER_PAYLOAD:

core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java

@@ -39,6 +39,7 @@
 import bisq.core.support.dispute.refund.RefundDisputeList;
 import bisq.core.trade.TradableList;
 import bisq.core.trade.statistics.TradeStatistics2Store;
+import bisq.core.trade.statistics.TradeStatistics3Store;
 import bisq.core.user.PreferencesPayload;
 import bisq.core.user.UserPayload;
 
@@ -126,6 +127,8 @@ public PersistableEnvelope fromProto(protobuf.PersistableEnvelope proto) {
                     return UnconfirmedBsqChangeOutputList.fromProto(proto.getUnconfirmedBsqChangeOutputList());
                 case SIGNED_WITNESS_STORE:
                     return SignedWitnessStore.fromProto(proto.getSignedWitnessStore());
+                case TRADE_STATISTICS3_STORE:
+                    return TradeStatistics3Store.fromProto(proto.getTradeStatistics3Store());
 
                 default:
                     throw new ProtobufferRuntimeException("Unknown proto message case(PB.PersistableEnvelope). " +

core/src/main/java/bisq/core/provider/fee/FeeService.java

@@ -52,7 +52,6 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-// TODO use dao parameters for fee
 @Slf4j
 public class FeeService {
 

core/src/main/java/bisq/core/provider/price/PriceFeedService.java

@@ -23,7 +23,7 @@
 import bisq.core.monetary.Price;
 import bisq.core.provider.PriceNodeHttpClient;
 import bisq.core.provider.ProvidersRepository;
-import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 import bisq.core.user.Preferences;
 
 import bisq.network.http.HttpClient;
@@ -50,6 +50,7 @@
 import java.time.Instant;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -288,12 +289,12 @@ public Date getLastRequestTimeStamp() {
         return new Date(epochInMillisAtLastRequest);
     }
 
-    public void applyLatestBisqMarketPrice(Set<TradeStatistics2> tradeStatisticsSet) {
+    public void applyLatestBisqMarketPrice(Set<TradeStatistics3> tradeStatisticsSet) {
         // takes about 10 ms for 5000 items
-        Map<String, List<TradeStatistics2>> mapByCurrencyCode = new HashMap<>();
+        Map<String, List<TradeStatistics3>> mapByCurrencyCode = new HashMap<>();
         tradeStatisticsSet.forEach(e -> {
-            final List<TradeStatistics2> list;
-            final String currencyCode = e.getCurrencyCode();
+            List<TradeStatistics3> list;
+            String currencyCode = e.getCurrency();
             if (mapByCurrencyCode.containsKey(currencyCode)) {
                 list = mapByCurrencyCode.get(currencyCode);
             } else {
@@ -306,9 +307,9 @@ public void applyLatestBisqMarketPrice(Set<TradeStatistics2> tradeStatisticsSet)
         mapByCurrencyCode.values().stream()
                 .filter(list -> !list.isEmpty())
                 .forEach(list -> {
-                    list.sort((o1, o2) -> o1.getTradeDate().compareTo(o2.getTradeDate()));
-                    TradeStatistics2 tradeStatistics = list.get(list.size() - 1);
-                    setBisqMarketPrice(tradeStatistics.getCurrencyCode(), tradeStatistics.getTradePrice());
+                    list.sort(Comparator.comparing(TradeStatistics3::getTradeDate));
+                    TradeStatistics3 tradeStatistics = list.get(list.size() - 1);
+                    setBisqMarketPrice(tradeStatistics.getCurrency(), tradeStatistics.getTradePrice());
                 });
     }
 

core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java

@@ -40,7 +40,8 @@ static void setSupportedCapabilities(Config config) {
                 Capability.SIGNED_ACCOUNT_AGE_WITNESS,
                 Capability.REFUND_AGENT,
                 Capability.TRADE_STATISTICS_HASH_UPDATE,
-                Capability.NO_ADDRESS_PRE_FIX
+                Capability.NO_ADDRESS_PRE_FIX,
+                Capability.TRADE_STATISTICS_3
         );
 
         if (config.daoActivated) {

core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java

@@ -157,7 +157,7 @@ public void cleanupDisputes() {
     @Override
     protected String getDisputeInfo(Dispute dispute) {
         String role = Res.get("shared.arbitrator").toLowerCase();
-        String link = "https://docs.bisq.network/trading-rules.html#legacy-arbitration"; //TODO needs to be created
+        String link = "https://docs.bisq.network/trading-rules.html#legacy-arbitration";
         return Res.get("support.initialInfo", role, role, link);
     }
 

core/src/main/java/bisq/core/trade/TradeModule.java

@@ -24,8 +24,6 @@
 import bisq.core.trade.closed.ClosedTradableManager;
 import bisq.core.trade.failed.FailedTradesManager;
 import bisq.core.trade.statistics.ReferralIdService;
-import bisq.core.trade.statistics.TradeStatistics2StorageService;
-import bisq.core.trade.statistics.TradeStatisticsManager;
 
 import bisq.common.app.AppModule;
 import bisq.common.config.Config;
@@ -46,8 +44,6 @@ public TradeModule(Config config) {
     @Override
     protected void configure() {
         bind(TradeManager.class).in(Singleton.class);
-        bind(TradeStatisticsManager.class).in(Singleton.class);
-        bind(TradeStatistics2StorageService.class).in(Singleton.class);
         bind(ClosedTradableManager.class).in(Singleton.class);
         bind(FailedTradesManager.class).in(Singleton.class);
         bind(AccountAgeWitnessService.class).in(Singleton.class);

core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java

@@ -24,7 +24,6 @@
 import bisq.core.trade.messages.PayoutTxPublishedMessage;
 import bisq.core.trade.messages.TradeMessage;
 import bisq.core.trade.protocol.tasks.ApplyFilter;
-import bisq.core.trade.protocol.tasks.PublishTradeStatistics;
 import bisq.core.trade.protocol.tasks.TradeTask;
 import bisq.core.trade.protocol.tasks.buyer.BuyerProcessDepositTxAndDelayedPayoutTxMessage;
 import bisq.core.trade.protocol.tasks.buyer.BuyerProcessPayoutTxPublishedMessage;
@@ -114,8 +113,7 @@ protected void handle(DepositTxAndDelayedPayoutTxMessage message, NodeAddress pe
                             removeMailboxMessageAfterProcessing(message);
                         }))
                 .setup(tasks(BuyerProcessDepositTxAndDelayedPayoutTxMessage.class,
-                        BuyerVerifiesFinalDelayedPayoutTx.class,
-                        PublishTradeStatistics.class)
+                        BuyerVerifiesFinalDelayedPayoutTx.class)
                         .using(new TradeTaskRunner(trade,
                                 () -> {
                                     stopTimeout();