PriceNode: Add support for multiple ExchangeRateProviders

build.gradle

@@ -57,7 +57,7 @@ configure(subprojects) {
         junitVersion = '4.12'
         jupiterVersion = '5.3.2'
         kotlinVersion = '1.3.41'
-        knowmXchangeVersion = '4.3.3'
+        knowmXchangeVersion = '4.4.2'
         langVersion = '3.8'
         logbackVersion = '1.1.11'
         loggingVersion = '1.2'
@@ -458,20 +458,43 @@ configure(project(':pricenode')) {
 
     dependencies {
         compile project(":core")
+
+        compileOnly "org.projectlombok:lombok:$lombokVersion"
+        annotationProcessor "org.projectlombok:lombok:$lombokVersion"
+
         implementation "com.google.code.gson:gson:$gsonVersion"
         implementation "commons-codec:commons-codec:$codecVersion"
         implementation "org.apache.httpcomponents:httpcore:$httpcoreVersion"
         implementation("org.apache.httpcomponents:httpclient:$httpclientVersion") {
             exclude(module: 'commons-codec')
         }
-        compile("org.knowm.xchange:xchange-bitcoinaverage:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-bitbay:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-btcmarkets:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-binance:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-bitfinex:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-bitflyer:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-bitstamp:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-cexio:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-coinmate:$knowmXchangeVersion")
         compile("org.knowm.xchange:xchange-coinmarketcap:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-coinone:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-exmo:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-hitbtc:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-huobi:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-independentreserve:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-kraken:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-luno:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-mercadobitcoin:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-paribu:$knowmXchangeVersion")
         compile("org.knowm.xchange:xchange-poloniex:$knowmXchangeVersion")
+        compile("org.knowm.xchange:xchange-quoine:$knowmXchangeVersion")
         compile("org.springframework.boot:spring-boot-starter-web:$springBootVersion")
         compile("org.springframework.boot:spring-boot-starter-actuator")
         testCompile "org.junit.jupiter:junit-jupiter-api:$jupiterVersion"
         testCompile "org.junit.jupiter:junit-jupiter-params:$jupiterVersion"
         testRuntime("org.junit.jupiter:junit-jupiter-engine:$jupiterVersion")
+        testCompileOnly "org.projectlombok:lombok:$lombokVersion"
+        testAnnotationProcessor "org.projectlombok:lombok:$lombokVersion"
     }
 
     test {

pricenode/README-HEROKU.md

@@ -5,7 +5,6 @@ Run the following commands:
 
     heroku create
     heroku buildpacks:add heroku/gradle
-    heroku config:set BITCOIN_AVG_PUBKEY=[your pubkey] BITCOIN_AVG_PRIVKEY=[your privkey]
     git push heroku master
     curl https://your-app-123456.herokuapp.com/getAllMarketPrices
 

pricenode/README.md

@@ -24,7 +24,6 @@ Operating a production pricenode is a valuable service to the Bisq network, and
 
 To run a pricenode, you will need:
 
-  - [BitcoinAverage API keys](https://bitcoinaverage.com/en/plans). Free plans are fine for local development or personal nodes; paid plans should be used for well-known production nodes.
   - JDK 8 if you want to build and run a node locally.
   - The `tor` binary (e.g. `brew install tor`) if you want to run a hidden service locally.
 
@@ -40,21 +39,6 @@ curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/ins
 
 At the end of the installer script, it should print your Tor onion hostname.
 
-### Setting your BitcoinAverage API keys
-
-Open `/etc/default/bisq-pricenode.env` in a text editor and look for these lines:
-```bash
-BITCOIN_AVG_PUBKEY=foo
-BITCOIN_AVG_PRIVKEY=bar
-```
-
-Add your pubkey and privkey and then reload/restart bisq-pricenode service:
-
-```bash
-systemctl daemon-reload
-systemctl restart bisq-pricenode
-```
-
 ### Test
 
 To manually test endpoints, run each of the following:

pricenode/bisq-pricenode.env

@@ -1,3 +1 @@
-BITCOIN_AVG_PUBKEY=foo
-BITCOIN_AVG_PRIVKEY=bar
 JAVA_OPTS=""

pricenode/docker/loop.sh

@@ -2,7 +2,7 @@
 while true
 do
 echo `date`  "(Re)-starting node"
-BITCOIN_AVG_PUBKEY=$BTCAVERAGE_PUBKEY BITCOIN_AVG_PRIVKEY=$BTCAVERAGE_PRIVKEY java -jar ./build/libs/bisq-pricenode.jar 2 2
+java -jar ./build/libs/bisq-pricenode.jar 2 2
 echo `date` "node terminated unexpectedly!!"
 sleep 3
 done

pricenode/install_pricenode_debian.sh

@@ -46,7 +46,7 @@ echo "[*] Adding Tor configuration"
 if ! grep "${BISQ_TORHS}" /etc/tor/torrc >/dev/null 2>&1;then
   sudo -H -i -u "${ROOT_USER}" sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${BISQ_TORHS}/ >> ${TOR_CONF}"
   sudo -H -i -u "${ROOT_USER}" sh -c "echo HiddenServicePort 80 127.0.0.1:8080 >> ${TOR_CONF}"
-  sudo -H -i -u "${ROOT_USER}" sh -c "echo HiddenServiceVersion 2 >> ${TOR_CONF}"
+  sudo -H -i -u "${ROOT_USER}" sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONF}"
 fi
 
 echo "[*] Creating Bisq user with Tor access"
@@ -60,8 +60,8 @@ echo "[*] Cloning Bisq repo"
 sudo -H -i -u "${BISQ_USER}" git config --global advice.detachedHead false
 sudo -H -i -u "${BISQ_USER}" git clone --branch "${BISQ_REPO_TAG}" "${BISQ_REPO_URL}" "${BISQ_HOME}/${BISQ_REPO_NAME}"
 
-echo "[*] Installing OpenJDK 10.0.2 from Bisq repo"
-sudo -H -i -u "${ROOT_USER}" "${BISQ_HOME}/${BISQ_REPO_NAME}/scripts/install_java.sh"
+echo "[*] Installing OpenJDK 11"
+sudo -H -i -u "${ROOT_USER}" apt-get install -qq -y openjdk-11-jdk
 
 echo "[*] Checking out Bisq ${BISQ_LATEST_RELEASE}"
 sudo -H -i -u "${BISQ_USER}" sh -c "cd ${BISQ_HOME}/${BISQ_REPO_NAME} && git checkout ${BISQ_LATEST_RELEASE}"

pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java

@@ -19,24 +19,53 @@
 
 import bisq.price.PriceProvider;
 
+import bisq.core.locale.CurrencyUtil;
+import bisq.core.locale.TradeCurrency;
+
+import org.knowm.xchange.Exchange;
+import org.knowm.xchange.ExchangeFactory;
+import org.knowm.xchange.currency.Currency;
+import org.knowm.xchange.currency.CurrencyPair;
+import org.knowm.xchange.dto.marketdata.Ticker;
+import org.knowm.xchange.exceptions.NotYetImplementedForExchangeException;
+import org.knowm.xchange.service.marketdata.MarketDataService;
+import org.knowm.xchange.service.marketdata.params.CurrencyPairsParam;
+import org.knowm.xchange.service.marketdata.params.Params;
+
 import java.time.Duration;
 
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Abstract base class for providers of bitcoin {@link ExchangeRate} data. Implementations
  * are marked with the {@link org.springframework.stereotype.Component} annotation in
- * order to be discovered via classpath scanning. Implementations are also marked with the
- * {@link org.springframework.core.annotation.Order} annotation to determine their
- * precedence over each other in the case of two or more services returning exchange rate
- * data for the same currency pair. In such cases, results from the provider with the
- * higher order value will take precedence over the provider with a lower value,
- * presuming that such providers are being iterated over in an ordered list.
+ * order to be discovered via classpath scanning. If multiple
+ * {@link ExchangeRateProvider}s retrieve rates for the same currency, then the
+ * {@link ExchangeRateService} will average them out and expose an aggregate rate.
  *
- * @see ExchangeRateService#ExchangeRateService(java.util.List)
+ * @see ExchangeRateService#getAllMarketPrices()
  */
 public abstract class ExchangeRateProvider extends PriceProvider<Set<ExchangeRate>> {
 
+    public static final Set<String> SUPPORTED_CRYPTO_CURRENCIES = CurrencyUtil.getAllSortedCryptoCurrencies().stream()
+            .map(TradeCurrency::getCode)
+            .collect(Collectors.toSet());
+
+    public static final Set<String> SUPPORTED_FIAT_CURRENCIES = CurrencyUtil.getAllSortedFiatCurrencies().stream()
+            .map(TradeCurrency::getCode)
+            .collect(Collectors.toSet());
+
     private final String name;
     private final String prefix;
 
@@ -60,4 +89,192 @@ protected void onRefresh() {
             .filter(e -> "USD".equals(e.getCurrency()) || "LTC".equals(e.getCurrency()))
             .forEach(e -> log.info("BTC/{}: {}", e.getCurrency(), e.getPrice()));
     }
+
+    /**
+     * @param exchangeClass Class of the {@link Exchange} for which the rates should be
+     *                      polled
+     * @return Exchange rates for Bisq-supported fiat currencies and altcoins in the
+     * specified {@link Exchange}
+     *
+     * @see CurrencyUtil#getAllSortedFiatCurrencies()
+     * @see CurrencyUtil#getAllSortedCryptoCurrencies()
+     */
+    protected Set<ExchangeRate> doGet(Class<? extends Exchange> exchangeClass) {
+        Set<ExchangeRate> result = new HashSet<ExchangeRate>();
+
+        // Initialize XChange objects
+        Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exchangeClass.getName());
+        MarketDataService marketDataService = exchange.getMarketDataService();
+
+        // Retrieve all currency pairs supported by the exchange
+        List<CurrencyPair> allCurrencyPairsOnExchange = exchange.getExchangeSymbols();
+
+        // Find out which currency pairs we are interested in polling ("desired pairs")
+        // This will be the intersection of:
+        // 1) the pairs available on the exchange, and
+        // 2) the pairs Bisq considers relevant / valid
+        // This will result in two lists of desired pairs (fiat and alts)
+
+        // Find the desired fiat pairs (pair format is BTC-FIAT)
+        List<CurrencyPair> desiredFiatPairs = allCurrencyPairsOnExchange.stream()
+                .filter(cp -> cp.base.equals(Currency.BTC))
+                .filter(cp -> SUPPORTED_FIAT_CURRENCIES.contains(cp.counter.getCurrencyCode()))
+                .collect(Collectors.toList());
+
+        // Find the desired altcoin pairs (pair format is ALT-BTC)
+        List<CurrencyPair> desiredCryptoPairs = allCurrencyPairsOnExchange.stream()
+                .filter(cp -> cp.counter.equals(Currency.BTC))
+                .filter(cp -> SUPPORTED_CRYPTO_CURRENCIES.contains(cp.base.getCurrencyCode()))
+                .collect(Collectors.toList());
+
+        // Retrieve in bulk all tickers offered by the exchange
+        // The benefits of this approach (vs polling each ticker) are twofold:
+        // 1) the polling of the exchange is faster (one HTTP call vs several)
+        // 2) it's easier to stay below any API rate limits the exchange might have
+        List<Ticker> tickersRetrievedFromExchange = new ArrayList<>();
+        try {
+            tickersRetrievedFromExchange = marketDataService.getTickers(new CurrencyPairsParam() {
+
+                /**
+                 * The {@link MarketDataService#getTickers(Params)} interface requires a
+                 * {@link CurrencyPairsParam} argument when polling for tickers in bulk.
+                 * This parameter is meant to indicate a list of currency pairs for which
+                 * the tickers should be polled. However, the actual implementations for
+                 * the different exchanges differ, for example:
+                 * - some will ignore it (and retrieve all available tickers)
+                 * - some will require it (and will fail if a null or empty list is given)
+                 * - some will properly handle it
+                 *
+                 * We take a simplistic approach, namely:
+                 * - for providers that require such a filter, specify one
+                 * - for all others, do not specify one
+                 *
+                 * We make this distinction using
+                 * {@link ExchangeRateProvider#requiresFilterDuringBulkTickerRetrieval}
+                 *
+                 * @return Filter (list of desired currency pairs) to be used during bulk
+                 * ticker retrieval
+                 */
+                @Override
+                public Collection<CurrencyPair> getCurrencyPairs() {
+                    // If required by the exchange implementation, specify a filter
+                    // (list of pairs which should be retrieved)
+                    if (requiresFilterDuringBulkTickerRetrieval())
+                        return Stream.of(desiredFiatPairs, desiredCryptoPairs)
+                                .flatMap(Collection::stream)
+                                .collect(Collectors.toList());
+
+                    // Otherwise, specify an empty list, indicating that the API should
+                    // simply return all available tickers
+                    return Collections.emptyList();
+                }
+            });
+
+            if (tickersRetrievedFromExchange.isEmpty()) {
+                // If the bulk ticker retrieval went through, but no tickers were
+                // retrieved, this is a strong indication that this specific exchange
+                // needs a specific list of pairs given as argument, for bulk retrieval to
+                // work. See requiresFilterDuringBulkTickerRetrieval()
+                throw new IllegalArgumentException("No tickers retrieved, " +
+                        "exchange requires explicit filter argument during bulk retrieval?");
+            }
+        }
+        catch (NotYetImplementedForExchangeException e) {
+            // Thrown when a provider has no marketDataService.getTickers() implementation
+            // either because the exchange API does not provide it, or because it has not
+            // been implemented yet in the knowm xchange library
+
+            // In this case (retrieval of bulk tickers is not possible) retrieve the
+            // tickers one by one
+            List<Ticker> finalTickersRetrievedFromExchange = tickersRetrievedFromExchange;
+            Stream.of(desiredFiatPairs, desiredCryptoPairs)
+                    .flatMap(Collection::stream)
+                    .collect(Collectors.toList())
+                    .forEach(cp -> {
+                        try {
+
+                            // This is done in a loop, and can therefore result in a burst
+                            // of API calls. Some exchanges do not allow bursts
+                            // A simplistic solution is to delay every call by 1 second
+                            // TODO Switch to using a more elegant solution (per exchange)
+                            // like ResilienceSpecification (needs knowm xchange libs v5)
+                            if (getMarketDataCallDelay() > 0) {
+                                Thread.sleep(getMarketDataCallDelay());
+                            }
+
+                            Ticker ticker = marketDataService.getTicker(cp);
+                            finalTickersRetrievedFromExchange.add(ticker);
+
+                        } catch (IOException | InterruptedException ioException) {
+                            ioException.printStackTrace();
+                            log.error("Could not query tickers for " + getName(), e);
+                        }
+                    });
+        }
+        catch (Exception e) {
+            // If there was a problem with polling this exchange, return right away,
+            // since there are no results to parse and process
+            log.error("Could not query tickers for provider " + getName(), e);
+            return result;
+        }
+
+        // Create an ExchangeRate for each desired currency pair ticker that was retrieved
+        Predicate<Ticker> isDesiredFiatPair = t -> desiredFiatPairs.contains(t.getCurrencyPair());
+        Predicate<Ticker> isDesiredCryptoPair = t -> desiredCryptoPairs.contains(t.getCurrencyPair());
+        tickersRetrievedFromExchange.stream()
+                .filter(isDesiredFiatPair.or(isDesiredCryptoPair)) // Only consider desired pairs
+                .forEach(t -> {
+                    // All tickers here match all requirements
+
+                    // We have two kinds of currency pairs, BTC-FIAT and ALT-BTC
+                    // In the first one, BTC is the first currency of the pair
+                    // In the second type, BTC is listed as the second currency
+                    // Distinguish between the two and create ExchangeRates accordingly
+
+                    // In every Bisq ExchangeRate, BTC is one currency in the pair
+                    // Extract the other currency from the ticker, to create ExchangeRates
+                    String otherExchangeRateCurrency;
+                    if (t.getCurrencyPair().base.equals(Currency.BTC)) {
+                        otherExchangeRateCurrency = t.getCurrencyPair().counter.getCurrencyCode();
+                    }
+                    else {
+                        otherExchangeRateCurrency = t.getCurrencyPair().base.getCurrencyCode();
+                    }
+
+                    result.add(new ExchangeRate(
+                            otherExchangeRateCurrency,
+                            t.getLast(),
+                            // Some exchanges do not provide timestamps
+                            t.getTimestamp() == null ? new Date() : t.getTimestamp(),
+                            this.getName()
+                    ));
+                });
+
+        return result;
+    }
+
+    /**
+     * Specifies optional delay between certain kind of API calls that can result in
+     * bursts. We want to avoid bursts, because this can cause certain exchanges to
+     * temporarily restrict access to the pricenode IP.
+     *
+     * @return Amount of milliseconds of delay between marketDataService.getTicker calls.
+     * By default 0, but can be overwritten by each provider.
+     */
+    protected long getMarketDataCallDelay() {
+        return 0;
+    }
+
+    /**
+     * @return Whether or not the bulk retrieval of tickers from the exchange requires an
+     * explicit filter (list of desired pairs) or not. If true, the
+     * {@link MarketDataService#getTickers(Params)} call will be constructed and given as
+     * argument, which acts as a filter indicating for which pairs the ticker should be
+     * retrieved. If false, {@link MarketDataService#getTickers(Params)} will be called
+     * with an empty argument, indicating that the API should simply return all available
+     * tickers on the exchange
+     */
+    protected boolean requiresFilterDuringBulkTickerRetrieval() {
+        return false;
+    }
 }