Skip to content

Commit

Permalink
Add redundancy to ARS/BTC pricenode:
Browse files Browse the repository at this point in the history
 - Remove ban over ARS fiat
 - ExchangeRateProvider by default does not consider rates with blue markets. CryptoYimplementation does.
 - ExchangeRateService uses ubiquous place for generating the rates / consider bluemarket prices / update tests accordingly
 - avoid procesing of blue markets if its already handled by EP / move bluelytics to its own util package
 - bluelytics util is singleton and updates gap async less frequently (1hr)
  • Loading branch information
rodvar committed Aug 11, 2023
1 parent a3cd9a1 commit c2b0c2d
Show file tree
Hide file tree
Showing 11 changed files with 262 additions and 39 deletions.
29 changes: 29 additions & 0 deletions src/main/java/bisq/price/spot/ExchangeRateProvider.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@
import org.knowm.xchange.service.marketdata.params.Params;
import org.springframework.core.env.Environment;

import javax.annotation.Nullable;
import java.io.IOException;
import java.math.BigDecimal;
import java.time.Duration;
import java.util.*;
import java.util.function.Predicate;
Expand Down Expand Up @@ -322,4 +324,31 @@ protected long getMarketDataCallDelay() {
protected boolean requiresFilterDuringBulkTickerRetrieval() {
return false;
}

/**
*
* @return true if the implementation of this ExchangeRateProvider already consider currencies
* blue markets if any. Defaults to false.
*/
public boolean alreadyConsidersBlueMarkets() {
return false;
}

/**
* @param originalRate original official rate for a certain currency. E.g. rate for ARS for Argentina PESO
* @return a new exchange rate if the implementation of this price provider already tackles real vs official
* market prices for the given currency.
* e.g. for FIAT ARS official rates are not the ones using in the free trading world.
* Most currencies won't need this, so defaults to null.
*/
public @Nullable ExchangeRate maybeUpdateBlueMarketPriceGapForRate(ExchangeRate originalRate, Double blueMarketGapForCurrency) {
if ("ARS".equalsIgnoreCase(originalRate.getCurrency())) {
double blueRate = originalRate.getPrice() * blueMarketGapForCurrency;
return new ExchangeRate(originalRate.getCurrency(),
BigDecimal.valueOf(blueRate),
new Date(originalRate.getTimestamp()),
originalRate.getProvider());
}
return null;
}
}
63 changes: 57 additions & 6 deletions src/main/java/bisq/price/spot/ExchangeRateService.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@

import bisq.core.util.InlierUtil;

import bisq.price.util.bluelytics.BlueLyticsUSDRate;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
Expand All @@ -36,7 +38,6 @@
@Service
class ExchangeRateService {
protected final Logger log = LoggerFactory.getLogger(this.getClass());

private final Environment env;
private final List<ExchangeRateProvider> providers;

Expand All @@ -57,9 +58,8 @@ public Map<String, Object> getAllMarketPrices() {
Map<String, ExchangeRate> aggregateExchangeRates = getAggregateExchangeRates();

providers.forEach(p -> {
if (p.get() == null)
return;
Set<ExchangeRate> exchangeRates = p.get();
Set<ExchangeRate> exchangeRates = providerCurrentExchangeRates(p);
if (exchangeRates == null) return;

// Specific metadata fields for specific providers are expected by the client,
// mostly for historical reasons
Expand All @@ -68,6 +68,7 @@ public Map<String, Object> getAllMarketPrices() {
metadata.putAll(getMetadata(p, exchangeRates));
});


LinkedHashMap<String, Object> result = new LinkedHashMap<>(metadata);
// Use a sorted list by currency code to make comparison of json data between
// different price nodes easier
Expand All @@ -78,6 +79,55 @@ public Map<String, Object> getAllMarketPrices() {
return result;
}

/**
* Please do not call provider.get(), use this method instead to consider currencies with blue markets
* @param provider to get the rates from
* @return the exchange rates for the different currencies the provider supports considering bluemarket rates if any
*/
public Set<ExchangeRate> providerCurrentExchangeRates(ExchangeRateProvider provider) {
Map<String, Double> blueMarketGapForCurrency = this.fetchBlueMarketGapForCurrencies();
Set<ExchangeRate> originalExchangeRates = provider.get();
if (originalExchangeRates == null)
return null;

Set<ExchangeRate> exchangeRates = new HashSet<>();

if (provider.alreadyConsidersBlueMarkets() ||
(Sets.intersection(originalExchangeRates.stream().map(ExchangeRate::getCurrency).collect(Collectors.toSet()),
blueMarketGapForCurrency.keySet())).isEmpty()) {
exchangeRates.addAll(originalExchangeRates);
} else {
this.addRatesUpdatingBlueGaps(blueMarketGapForCurrency, provider, exchangeRates, originalExchangeRates);
}
return exchangeRates;
}

private void addRatesUpdatingBlueGaps(Map<String, Double> blueMarketGapForCurrency, ExchangeRateProvider provider, Set<ExchangeRate> exchangeRates, Set<ExchangeRate> originalExchangeRates) {
originalExchangeRates.forEach(er -> {
// default to original rate
ExchangeRate exchangeRateToUse = er;
if (blueMarketGapForCurrency.containsKey(er.getCurrency())) {
ExchangeRate updatedExchangeRate = provider.maybeUpdateBlueMarketPriceGapForRate(er, blueMarketGapForCurrency.get(er.getCurrency()));
if (updatedExchangeRate != null) {
exchangeRateToUse = updatedExchangeRate;
// this.log.info(String.format("Replaced original %s rate of $%s to $%s", er.getCurrency(), BigDecimal.valueOf(er.getPrice()).toEngineeringString(), BigDecimal.valueOf(updatedExchangeRate.getPrice()).toEngineeringString()));
}
}
exchangeRates.add(exchangeRateToUse);
});
}

private Map<String, Double> fetchBlueMarketGapForCurrencies() {
Map<String, Double> blueMarketGapForCurrencies = new HashMap<>();
// ARS
Double arsBlueMultiplier = BlueLyticsUSDRate.getInstance().blueGapMultiplier();
if (!arsBlueMultiplier.isNaN()) {
// this.log.info("Updated ARS/USD multiplier is " + arsBlueMultiplier);
blueMarketGapForCurrencies.put("ARS", arsBlueMultiplier);
}
return blueMarketGapForCurrencies;
}

/**
* For each currency, create an aggregate {@link ExchangeRate} based on the currency's
* rates from all providers. If multiple providers have rates for the currency, then
Expand Down Expand Up @@ -164,9 +214,10 @@ private double getOutlierStdDeviation() {
private Map<String, List<ExchangeRate>> getCurrencyCodeToExchangeRates() {
Map<String, List<ExchangeRate>> currencyCodeToExchangeRates = new HashMap<>();
for (ExchangeRateProvider p : providers) {
if (p.get() == null)
Set<ExchangeRate> exchangeRates = providerCurrentExchangeRates(p);
if (exchangeRates == null)
continue;
for (ExchangeRate exchangeRate : p.get()) {
for (ExchangeRate exchangeRate : exchangeRates) {
String currencyCode = exchangeRate.getCurrency();
if (currencyCodeToExchangeRates.containsKey(currencyCode)) {
List<ExchangeRate> l = new ArrayList<>(currencyCodeToExchangeRates.get(currencyCode));
Expand Down
28 changes: 13 additions & 15 deletions src/main/java/bisq/price/spot/providers/CoinGecko.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@

package bisq.price.spot.providers;

import bisq.asset.Coin;
import bisq.price.spot.ExchangeRate;
import bisq.price.spot.ExchangeRateProvider;
import bisq.price.util.coingecko.CoinGeckoMarketData;

import bisq.price.util.coingecko.CoinGeckoTicker;
import lombok.Getter;
import lombok.Setter;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.core.env.Environment;
import org.springframework.http.RequestEntity;
Expand All @@ -33,16 +37,13 @@
import java.math.BigDecimal;
import java.math.RoundingMode;

import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;

@Component
class CoinGecko extends ExchangeRateProvider {

private static final String GET_EXCHANGE_RATES_URL = "https://api.coingecko.com/api/v3/exchange_rates";
private final RestTemplate restTemplate = new RestTemplate();

public CoinGecko(Environment env) {
Expand All @@ -60,18 +61,16 @@ public Set<ExchangeRate> doGet() {
Predicate<Map.Entry> isDesiredFiatPair = t -> getSupportedFiatCurrencies().contains(t.getKey());
Predicate<Map.Entry> isDesiredCryptoPair = t -> getSupportedCryptoCurrencies().contains(t.getKey());

getMarketData().getRates().entrySet().stream()
Map<String, CoinGeckoTicker> rates = getMarketData().getRates();
rates.entrySet().stream()
.filter(isDesiredFiatPair.or(isDesiredCryptoPair))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
.forEach((key, ticker) -> {

boolean useInverseRate = false;
if (getSupportedCryptoCurrencies().contains(key)) {
// Use inverse rate for alts, because the API returns the
// conversion rate in the opposite direction than what we need
// API returns the BTC/Alt rate, we need the Alt/BTC rate
useInverseRate = true;
}
boolean useInverseRate = getSupportedCryptoCurrencies().contains(key);
// Use inverse rate for alts, because the API returns the
// conversion rate in the opposite direction than what we need
// API returns the BTC/Alt rate, we need the Alt/BTC rate

BigDecimal rate = ticker.getValue();
// Find the inverse rate, while using enough decimals to reflect very
Expand All @@ -87,15 +86,14 @@ public Set<ExchangeRate> doGet() {
this.getName()
));
});

return result;
}

private CoinGeckoMarketData getMarketData() {
return restTemplate.exchange(
RequestEntity
.get(UriComponentsBuilder
.fromUriString("https://api.coingecko.com/api/v3/exchange_rates").build()
.fromUriString(CoinGecko.GET_EXCHANGE_RATES_URL).build()
.toUri())
.build(),
new ParameterizedTypeReference<CoinGeckoMarketData>() {
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/bisq/price/spot/providers/CryptoYa.java
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ public CryptoYa(Environment env) {
super(env, "CRYPTOYA", "cryptoya", Duration.ofMinutes(1));
}

@Override
public boolean alreadyConsidersBlueMarkets() {
return true;
}

/**
*
* @return average price buy/sell price averaging different providers suported by cryptoya api
Expand Down
86 changes: 86 additions & 0 deletions src/main/java/bisq/price/util/bluelytics/BlueLyticsUSDRate.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package bisq.price.util.bluelytics;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.RequestEntity;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;

import java.time.Instant;
import java.util.Date;
import java.util.Objects;

/**
* Util singleton object to update and provider ARS/USD blue market gap against the oficial rate.
* This is useful for example, to estimate ARS/BTC blue (real) market rate in a country with heavy currency controls
*/
public final class BlueLyticsUSDRate {
private static final long MIN_REFRESH_WINDOW = 3600000; // 1hr
private static final String GET_USD_EXCHANGE_RATES_ARG_URL = "https://api.bluelytics.com.ar/v2/latest";
private static final BlueLyticsUSDRate instance = new BlueLyticsUSDRate();
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private final RestTemplate restTemplate = new RestTemplate();
private Double lastBlueGap;
private Long lastRefresh;

private BlueLyticsUSDRate() {
lastRefresh = null;
lastBlueGap = null;
}

public static BlueLyticsUSDRate getInstance() {
return BlueLyticsUSDRate.instance;
}

/**
*
* @return current ARS/USD gap multiplier to get from official rate to free market rate.
* If not available returns Nan
*/
public Double blueGapMultiplier() {
if (this.lastRefresh == null) {
this.refreshBlueGap();
} else {
this.maybeLaunchAsyncRefresh();
}
return Objects.requireNonNullElse(this.lastBlueGap, Double.NaN);
}

/**
* if enough time {@see BlueLyticsUSDRate.MIN_FRESH_WINDOW} has pass from the last refresh, launch async refresh
*/
private void maybeLaunchAsyncRefresh() {
if (this.lastRefresh != null &&
Date.from(Instant.now()).getTime() > this.lastRefresh + BlueLyticsUSDRate.MIN_REFRESH_WINDOW) {
this.launchAsyncRefresh();
}
}

private synchronized void launchAsyncRefresh() {
new Thread(this::refreshBlueGap).start();
}

private void refreshBlueGap() {
try {
// the last_update value is different than the last one and also launch the update if 1 hour passed ?
this.lastBlueGap = Objects.requireNonNull(restTemplate.exchange(
RequestEntity
.get(UriComponentsBuilder
.fromUriString(BlueLyticsUSDRate.GET_USD_EXCHANGE_RATES_ARG_URL).build()
.toUri())
.build(),
new ParameterizedTypeReference<BlueLyticsUsdRates>() {
}
).getBody()).gapSellMultiplier();
this.lastRefresh = new Date().getTime();
} catch (Exception e) {
this.logger.error("Failed to fetch updated bluelytics gap multiplier", e);
}
}

@Override
protected Object clone() throws CloneNotSupportedException {
throw new CloneNotSupportedException("Cannot clone Singleton");
}
}
31 changes: 31 additions & 0 deletions src/main/java/bisq/price/util/bluelytics/BlueLyticsUsdRates.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package bisq.price.util.bluelytics;

import lombok.Getter;
import lombok.Setter;

import java.util.Date;

@Getter
@Setter
public class BlueLyticsUsdRates {
@Getter
@Setter
public static class USDRate {
Double value_avg;
Double value_sell;
Double value_buy;
}

BlueLyticsUsdRates.USDRate oficial;
BlueLyticsUsdRates.USDRate blue;
Date last_update;

/**
*
* @return the sell multiplier to go from oficial to blue market for ARS/USD
* if its not available, returns NaN
*/
public Double gapSellMultiplier() {
return this.blue.value_sell / this.oficial.value_sell;
}
}
2 changes: 1 addition & 1 deletion src/main/resources/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ bisq.price.mining.providers.mempoolHostname.2=mempool.emzy.de
bisq.price.mining.providers.mempoolHostname.3=mempool.ninja
bisq.price.mining.providers.mempoolHostname.4=mempool.bisq.services
# bisq.price.mining.providers.mempoolHostname.5=someHostOrIP
bisq.price.fiatcurrency.excluded=LBP,ARS
bisq.price.fiatcurrency.excluded=LBP
bisq.price.fiatcurrency.excludedByProvider=HUOBI:BRL
bisq.price.cryptocurrency.excluded=
bisq.price.outlierStdDeviation=2.2
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,20 @@ private void checkProviderCurrencyPairs(ExchangeRateProvider exchangeProvider, S
.collect(Collectors.toSet());

Set<String> supportedFiatCurrenciesRetrieved = exchangeProvider.getSupportedFiatCurrencies().stream()
.filter(f -> retrievedRatesCurrencies.contains(f))
.filter(retrievedRatesCurrencies::contains)
.collect(Collectors.toCollection(TreeSet::new));
log.info("Retrieved rates for supported fiat currencies: " + supportedFiatCurrenciesRetrieved);

Set<String> supportedCryptoCurrenciesRetrieved = exchangeProvider.getSupportedCryptoCurrencies().stream()
.filter(c -> retrievedRatesCurrencies.contains(c))
.filter(retrievedRatesCurrencies::contains)
.collect(Collectors.toCollection(TreeSet::new));
log.info("Retrieved rates for supported altcoins: " + supportedCryptoCurrenciesRetrieved);

Set<String> supportedCurrencies = Sets.union(
exchangeProvider.getSupportedCryptoCurrencies(),
exchangeProvider.getSupportedFiatCurrencies());

Set unsupportedCurrencies = Sets.difference(retrievedRatesCurrencies, supportedCurrencies);
Set<String> unsupportedCurrencies = Sets.difference(retrievedRatesCurrencies, supportedCurrencies);
assertTrue(unsupportedCurrencies.isEmpty(),
"Retrieved exchange rates contain unsupported currencies: " + unsupportedCurrencies);
}
Expand Down
Loading

0 comments on commit c2b0c2d

Please sign in to comment.