From 2d00f255942bb81d89e69841551226815dcc5e95 Mon Sep 17 00:00:00 2001 From: jmacxx <47253594+jmacxx@users.noreply.github.com> Date: Sat, 14 Oct 2023 22:32:04 -0500 Subject: [PATCH] Remove stale prices that have not been updated for over 10 minutes. --- src/main/java/bisq/price/PriceProvider.java | 6 +++- .../bisq/price/spot/ExchangeRateProvider.java | 18 ++++++++++ .../bisq/price/spot/ExchangeRateService.java | 1 + .../price/spot/ExchangeRateServiceTest.java | 34 +++++++++++++++++-- 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/main/java/bisq/price/PriceProvider.java b/src/main/java/bisq/price/PriceProvider.java index cbeefd9..f4e4981 100644 --- a/src/main/java/bisq/price/PriceProvider.java +++ b/src/main/java/bisq/price/PriceProvider.java @@ -51,6 +51,10 @@ public final T get() { return cachedResult; } + public final void put(T values) { + cachedResult = values; + } + @Override public final void start() { // do the initial refresh asynchronously @@ -80,7 +84,7 @@ public void run() { private void refresh() { long ts = System.currentTimeMillis(); - cachedResult = doGet(); + put(doGet()); log.info("refresh took {} ms.", (System.currentTimeMillis() - ts)); diff --git a/src/main/java/bisq/price/spot/ExchangeRateProvider.java b/src/main/java/bisq/price/spot/ExchangeRateProvider.java index 5dafb61..56fe747 100644 --- a/src/main/java/bisq/price/spot/ExchangeRateProvider.java +++ b/src/main/java/bisq/price/spot/ExchangeRateProvider.java @@ -39,6 +39,7 @@ import java.math.RoundingMode; import java.time.Duration; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -54,6 +55,7 @@ */ public abstract class ExchangeRateProvider extends PriceProvider> { + private static final long STALE_PRICE_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(10); private static Set SUPPORTED_CRYPTO_CURRENCIES = new HashSet<>(); private static Set SUPPORTED_FIAT_CURRENCIES = new HashSet<>(); private final Set providerExclusionList = new HashSet<>(); @@ -131,6 +133,22 @@ public String getPrefix() { return prefix; } + public void maybeClearStaleRates() { + // a stale rate is older than the specified interval, except: + // timestamp of 0L is used as special case re: CoinMarketCap and BitcoinAverage + // (https://github.com/bisq-network/bisq-pricenode/issues/23) + long staleTimestamp = new Date().getTime() - STALE_PRICE_INTERVAL_MILLIS; + Set nonStaleRates = get().stream() + .filter(e -> e.getTimestamp() == 0L || e.getTimestamp() > staleTimestamp) + .collect(Collectors.toSet()); + long numberOriginalRates = get().size(); + if (numberOriginalRates > nonStaleRates.size()) { + put(nonStaleRates); + log.warn("{} {} stale rates removed, now {} rates", + getName(), numberOriginalRates, nonStaleRates.size()); + } + } + @Override protected void onRefresh() { get().stream() diff --git a/src/main/java/bisq/price/spot/ExchangeRateService.java b/src/main/java/bisq/price/spot/ExchangeRateService.java index 7b48f21..ff8322b 100644 --- a/src/main/java/bisq/price/spot/ExchangeRateService.java +++ b/src/main/java/bisq/price/spot/ExchangeRateService.java @@ -60,6 +60,7 @@ public Map getAllMarketPrices() { Map aggregateExchangeRates = getAggregateExchangeRates(); providers.forEach(p -> { + p.maybeClearStaleRates(); // Specific metadata fields for specific providers are expected by the client, // mostly for historical reasons // Therefore, add metadata fields for all known providers diff --git a/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java b/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java index a5bc01b..45796bd 100644 --- a/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java +++ b/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java @@ -165,6 +165,28 @@ public void getAllMarketPrices_withMultipleProviders_overlappingCurrencyCodes() assertNotEquals(0L, retrievedData.get(providers.get(1).getPrefix() + "Ts")); } + @Test + public void testStaleRatesRemoved() { + String fiatCoin = "BRL"; + // insert a stale rate + long staleTimestamp = 10000L; + Long validTimestamp = System.currentTimeMillis(); + List providers = asList( + buildDummyExchangeRateProviderWithRateAndTimestamp("mercadoBitcoin", fiatCoin, 129000.0, staleTimestamp), + buildDummyExchangeRateProviderWithRateAndTimestamp("coinGecko", fiatCoin, 129000.0, validTimestamp), + buildDummyExchangeRateProviderWithRateAndTimestamp("binance", fiatCoin, 131000.0, validTimestamp)); + Map retrievedData = new ExchangeRateService(new StandardEnvironment(), providers, Collections.emptyList()).getAllMarketPrices(); + doSanityChecksForRetrievedDataMultipleProviders(retrievedData, providers); + // check that the provider's stale rate was removed resulting in 0 rates and a 0 timestamp + assertEquals("0", retrievedData.get("mercadoBitcoinTs").toString()); + assertEquals("0", retrievedData.get("mercadoBitcoinCount").toString()); + // check that other providers are unaffected + assertEquals("1", retrievedData.get("coinGeckoCount").toString()); + assertEquals("1", retrievedData.get("binanceCount").toString()); + assertEquals(validTimestamp.toString(), retrievedData.get("coinGeckoTs").toString()); + assertEquals(validTimestamp.toString(), retrievedData.get("binanceTs").toString()); + } + @Test public void bisqIndexCalculation_oneOutlierPriceWideRange() { String fiatCoin = "BRL"; @@ -517,7 +539,13 @@ protected Set doGet() { return dummyProvider; } - private ExchangeRateProvider buildDummyExchangeRateProviderWithRate(String providerName, String currencyCode, Double dummyRate) { + private ExchangeRateProvider buildDummyExchangeRateProviderWithRate( + String providerName, String currencyCode, Double rate) { + return buildDummyExchangeRateProviderWithRateAndTimestamp( + providerName, currencyCode, rate, System.currentTimeMillis()); + } + private ExchangeRateProvider buildDummyExchangeRateProviderWithRateAndTimestamp( + String providerName, String currencyCode, Double rate, long timestamp) { ExchangeRateProvider dummyProvider = new ExchangeRateProvider( new StandardEnvironment(), providerName, @@ -534,8 +562,8 @@ protected Set doGet() { HashSet exchangeRates = new HashSet<>(); exchangeRates.add(new ExchangeRate( currencyCode, - dummyRate, - System.currentTimeMillis(), + rate, + timestamp, getName())); // ExchangeRateProvider name return exchangeRates; }