Merge pull request #29 from jmacxx/improve_outlier_filtering

Switch outlier price filtering to 1.1 standard deviation.
bisq-network · Sep 15, 2023 · 806cdbf · 806cdbf
2 parents 94b2674 + ff2bed7
commit 806cdbf
Show file tree

Hide file tree

Showing 3 changed files with 228 additions and 98 deletions.
diff --git a/src/main/java/bisq/price/spot/ExchangeRateService.java b/src/main/java/bisq/price/spot/ExchangeRateService.java
@@ -110,51 +110,57 @@ private Map<String, ExchangeRate> getAggregateExchangeRates() {
             } else {
                 // If multiple providers have rates for this currency, then
                 // aggregate = average of the rates
-                List<Double> goodPriceList = removeOutliers(exchangeRateList.stream().
-                        mapToDouble(ExchangeRate::getPrice).boxed().collect(Collectors.toList()), currencyCode);
-                OptionalDouble opt = goodPriceList.stream().mapToDouble(Double::doubleValue).average();
-                // List size > 1, so opt is always set
-                double priceAvg = opt.orElseThrow(IllegalStateException::new);
+                double priceAvg = priceAverageWithOutliersRemoved(exchangeRateList, currencyCode);
                 aggregateExchangeRate = new ExchangeRate(
                         currencyCode,
                         BigDecimal.valueOf(priceAvg),
                         new Date(), // timestamp = time when avg is calculated
                         "Bisq-Aggregate");
-                // log the outlier prices which were removed from the average, if any.
-                for (ExchangeRate badRate : exchangeRateList.stream()
-                        .filter(e -> !goodPriceList.contains(e.getPrice()))
-                        .collect(Collectors.toList())) {
-                    log.warn("outlier price removed={}, source={}, ccy={}, consensus price={}",
-                            badRate.getPrice(),
-                            badRate.getProvider(),
-                            currencyCode,
-                            aggregateExchangeRate.getPrice());
-                }
             }
             aggregateExchangeRates.put(aggregateExchangeRate.getCurrency(), aggregateExchangeRate);
         });
 
         return aggregateExchangeRates;
     }
 
-    private List<Double> removeOutliers(List<Double> yValues, String contextInfo) {
+    private double priceAverageWithOutliersRemoved(List<ExchangeRate> exchangeRateList, String contextInfo) {
+        final List<Double> yValues = exchangeRateList.stream().
+                mapToDouble(ExchangeRate::getPrice).boxed().collect(Collectors.toList());
         Tuple2<Double, Double> tuple = InlierUtil.findInlierRange(yValues, 0, getOutlierStdDeviation());
         double lowerBound = tuple.first;
         double upperBound = tuple.second;
-        List<Double> filteredPrices = yValues.stream()
-                .filter(e -> e >= lowerBound)
-                .filter(e -> e <= upperBound)
+        final List<ExchangeRate> filteredPrices = exchangeRateList.stream()
+                .filter(e -> e.getPrice() >= lowerBound)
+                .filter(e -> e.getPrice() <= upperBound)
                 .collect(Collectors.toList());
+
         if (filteredPrices.size() < 1) {
-            log.error("{}: no results after outliers removed. lowerBound={}, upperBound={}, stdDev={}, yValues={}",
-                    contextInfo, lowerBound, upperBound, getOutlierStdDeviation(), yValues.toString());
-            return yValues;   // all prices cannot be removed, so revert to keep service running
+            log.error("{}: could not filter, revert to plain average. lowerBound={}, upperBound={}, stdDev={}, yValues={}",
+                    contextInfo, lowerBound, upperBound, getOutlierStdDeviation(), yValues);
+            return exchangeRateList.stream().mapToDouble(ExchangeRate::getPrice).average().getAsDouble();
+        }
+
+        OptionalDouble opt = filteredPrices.stream().mapToDouble(ExchangeRate::getPrice).average();
+        // List size > 1, so opt is always set
+        double priceAvg = opt.orElseThrow(IllegalStateException::new);
+
+        // log the outlier prices which were removed from the average, if any.
+        for (ExchangeRate badRate : exchangeRateList.stream()
+                .filter(e -> !filteredPrices.contains(e))
+                .collect(Collectors.toList())) {
+            log.info("{} {} outlier price removed:{}, lower/upper bounds:{}/{}, consensus price:{}",
+                    badRate.getProvider(),
+                    badRate.getCurrency(),
+                    badRate.getPrice(),
+                    lowerBound,
+                    upperBound,
+                    priceAvg);
         }
-        return filteredPrices;
+        return priceAvg;
     }
 
     private double getOutlierStdDeviation() {
-        return Double.parseDouble(env.getProperty("bisq.price.outlierStdDeviation", "2.2"));
+        return Double.parseDouble(env.getProperty("bisq.price.outlierStdDeviation", "1.1"));
     }
 
     /**

diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
@@ -10,4 +10,4 @@ bisq.price.mining.providers.mempoolHostname.4=mempool.bisq.services
 bisq.price.fiatcurrency.excluded=LBP
 bisq.price.fiatcurrency.excludedByProvider=HUOBI:BRL
 bisq.price.cryptocurrency.excluded=
-bisq.price.outlierStdDeviation=2.2
+bisq.price.outlierStdDeviation=1.1