From 4ad1f4d246b808d2a08f7af0e52b680724629843 Mon Sep 17 00:00:00 2001 From: HenrikJannsen Date: Thu, 24 Nov 2022 16:26:31 -0500 Subject: [PATCH 1/4] Remove pricenode module It has been extracted to https://github.com/bisq-network/bisq-pricenode Signed-off-by: HenrikJannsen --- Makefile | 1 - build.gradle | 77 ---- pricenode/Procfile | 1 - pricenode/README-HEROKU.md | 42 -- pricenode/README.md | 86 ---- pricenode/TODO.md | 20 - pricenode/bisq-pricenode.env | 1 - pricenode/bisq-pricenode.service | 22 - pricenode/collectd.conf.snippet | 5 - pricenode/docker/Dockerfile | 26 -- pricenode/docker/README.md | 43 -- pricenode/docker/cloudconfig.yml | 103 ----- pricenode/docker/docker-compose.yml | 21 - pricenode/docker/installDockerCompose.sh | 4 - pricenode/docker/loop.sh | 8 - pricenode/docker/rebuildAndRestart.sh | 4 - pricenode/docker/start_node.sh | 1 - pricenode/docker/start_tor.sh | 4 - pricenode/docker/torrc | 2 - pricenode/install_hsversion_debian.sh | 41 -- pricenode/install_networksize_debian.sh | 42 -- pricenode/install_pricenode_debian.sh | 90 ---- pricenode/journalscraper.sh | 20 - pricenode/journalscraper_hsversion.sh | 20 - pricenode/src/main/java/bisq/price/Main.java | 51 --- .../main/java/bisq/price/PriceController.java | 35 -- .../main/java/bisq/price/PriceProvider.java | 120 ----- .../main/java/bisq/price/mining/FeeRate.java | 52 --- .../bisq/price/mining/FeeRateController.java | 40 -- .../bisq/price/mining/FeeRateProvider.java | 36 -- .../bisq/price/mining/FeeRateService.java | 107 ----- .../providers/MempoolFeeRateProvider.java | 228 ---------- .../java/bisq/price/spot/ExchangeRate.java | 99 ----- .../price/spot/ExchangeRateController.java | 58 --- .../bisq/price/spot/ExchangeRateProvider.java | 338 -------------- .../bisq/price/spot/ExchangeRateService.java | 187 -------- .../bisq/price/spot/providers/BTCMarkets.java | 46 -- .../bisq/price/spot/providers/Binance.java | 46 -- .../price/spot/providers/BitcoinAverage.java | 57 --- .../bisq/price/spot/providers/Bitfinex.java | 45 -- .../bisq/price/spot/providers/Bitflyer.java | 45 -- .../bisq/price/spot/providers/Bitstamp.java | 45 -- .../bisq/price/spot/providers/CoinGecko.java | 105 ----- .../price/spot/providers/CoinMarketCap.java | 54 --- .../price/spot/providers/CoinbasePro.java | 49 --- .../bisq/price/spot/providers/Coinone.java | 46 -- .../spot/providers/IndependentReserve.java | 46 -- .../bisq/price/spot/providers/Kraken.java | 50 --- .../java/bisq/price/spot/providers/Luno.java | 53 --- .../price/spot/providers/MercadoBitcoin.java | 46 -- .../bisq/price/spot/providers/Paribu.java | 45 -- .../bisq/price/spot/providers/Poloniex.java | 45 -- .../bisq/price/spot/providers/Quoine.java | 50 --- .../main/java/bisq/price/util/Altcoins.java | 32 -- .../price/util/bitpay/BitpayMarketData.java | 13 - .../bisq/price/util/bitpay/BitpayTicker.java | 18 - .../util/coingecko/CoinGeckoMarketData.java | 22 - .../price/util/coingecko/CoinGeckoTicker.java | 20 - .../coinpaprika/CoinpaprikaMarketData.java | 17 - .../util/coinpaprika/CoinpaprikaTicker.java | 16 - .../src/main/resources/application.properties | 12 - pricenode/src/main/resources/banner.txt | 6 - pricenode/src/main/resources/logback.xml | 16 - pricenode/src/main/resources/version.txt | 1 - .../AbstractExchangeRateProviderTest.java | 66 --- .../bisq/price/mining/FeeRateServiceTest.java | 110 ----- .../providers/MempoolFeeRateProviderTest.java | 100 ----- .../price/spot/ExchangeRateServiceTest.java | 414 ------------------ .../price/spot/providers/BTCMarketsTest.java | 36 -- .../price/spot/providers/BinanceTest.java | 36 -- .../price/spot/providers/BitfinexTest.java | 36 -- .../price/spot/providers/BitflyerTest.java | 36 -- .../price/spot/providers/BitstampTest.java | 36 -- .../price/spot/providers/CoinGeckoTest.java | 36 -- .../price/spot/providers/CoinbaseProTest.java | 36 -- .../price/spot/providers/CoinoneTest.java | 36 -- .../providers/IndependentReserveTest.java | 36 -- .../bisq/price/spot/providers/KrakenTest.java | 36 -- .../bisq/price/spot/providers/LunoTest.java | 36 -- .../spot/providers/MercadoBitcoinTest.java | 36 -- .../bisq/price/spot/providers/ParibuTest.java | 36 -- .../price/spot/providers/PoloniexTest.java | 36 -- .../bisq/price/spot/providers/QuoineTest.java | 36 -- pricenode/torrc | 2 - pricenode/uninstall_hsversion_debian.sh | 29 -- settings.gradle | 1 - 86 files changed, 4343 deletions(-) delete mode 100644 pricenode/Procfile delete mode 100644 pricenode/README-HEROKU.md delete mode 100644 pricenode/README.md delete mode 100644 pricenode/TODO.md delete mode 100644 pricenode/bisq-pricenode.env delete mode 100644 pricenode/bisq-pricenode.service delete mode 100644 pricenode/collectd.conf.snippet delete mode 100644 pricenode/docker/Dockerfile delete mode 100644 pricenode/docker/README.md delete mode 100644 pricenode/docker/cloudconfig.yml delete mode 100644 pricenode/docker/docker-compose.yml delete mode 100644 pricenode/docker/installDockerCompose.sh delete mode 100644 pricenode/docker/loop.sh delete mode 100755 pricenode/docker/rebuildAndRestart.sh delete mode 100644 pricenode/docker/start_node.sh delete mode 100644 pricenode/docker/start_tor.sh delete mode 100644 pricenode/docker/torrc delete mode 100755 pricenode/install_hsversion_debian.sh delete mode 100755 pricenode/install_networksize_debian.sh delete mode 100755 pricenode/install_pricenode_debian.sh delete mode 100755 pricenode/journalscraper.sh delete mode 100755 pricenode/journalscraper_hsversion.sh delete mode 100644 pricenode/src/main/java/bisq/price/Main.java delete mode 100644 pricenode/src/main/java/bisq/price/PriceController.java delete mode 100644 pricenode/src/main/java/bisq/price/PriceProvider.java delete mode 100644 pricenode/src/main/java/bisq/price/mining/FeeRate.java delete mode 100644 pricenode/src/main/java/bisq/price/mining/FeeRateController.java delete mode 100644 pricenode/src/main/java/bisq/price/mining/FeeRateProvider.java delete mode 100644 pricenode/src/main/java/bisq/price/mining/FeeRateService.java delete mode 100644 pricenode/src/main/java/bisq/price/mining/providers/MempoolFeeRateProvider.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/ExchangeRate.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/ExchangeRateController.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/ExchangeRateService.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/BTCMarkets.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Binance.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/BitcoinAverage.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Bitfinex.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Bitflyer.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Bitstamp.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/CoinGecko.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/CoinMarketCap.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/CoinbasePro.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Coinone.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/IndependentReserve.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Kraken.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Luno.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/MercadoBitcoin.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Paribu.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Poloniex.java delete mode 100644 pricenode/src/main/java/bisq/price/spot/providers/Quoine.java delete mode 100644 pricenode/src/main/java/bisq/price/util/Altcoins.java delete mode 100644 pricenode/src/main/java/bisq/price/util/bitpay/BitpayMarketData.java delete mode 100644 pricenode/src/main/java/bisq/price/util/bitpay/BitpayTicker.java delete mode 100644 pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoMarketData.java delete mode 100644 pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoTicker.java delete mode 100644 pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaMarketData.java delete mode 100644 pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaTicker.java delete mode 100644 pricenode/src/main/resources/application.properties delete mode 100644 pricenode/src/main/resources/banner.txt delete mode 100644 pricenode/src/main/resources/logback.xml delete mode 100644 pricenode/src/main/resources/version.txt delete mode 100644 pricenode/src/test/java/bisq/price/AbstractExchangeRateProviderTest.java delete mode 100644 pricenode/src/test/java/bisq/price/mining/FeeRateServiceTest.java delete mode 100644 pricenode/src/test/java/bisq/price/mining/providers/MempoolFeeRateProviderTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/BTCMarketsTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/BinanceTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/BitfinexTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/BitflyerTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/BitstampTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/CoinGeckoTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/CoinbaseProTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/CoinoneTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/IndependentReserveTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/KrakenTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/LunoTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/MercadoBitcoinTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/ParibuTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/PoloniexTest.java delete mode 100644 pricenode/src/test/java/bisq/price/spot/providers/QuoineTest.java delete mode 100644 pricenode/torrc delete mode 100755 pricenode/uninstall_hsversion_debian.sh diff --git a/Makefile b/Makefile index 9fac1408c9d..58ff18290b2 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,6 @@ # $ ls -1 bisq-* # bisq-desktop # bisq-monitor -# bisq-pricenode # bisq-relay # bisq-seednode # bisq-statsnode diff --git a/build.gradle b/build.gradle index 7697799c905..aebb42af476 100644 --- a/build.gradle +++ b/build.gradle @@ -61,7 +61,6 @@ configure(subprojects) { junitVersion = '4.12' jupiterVersion = '5.7.0' kotlinVersion = '1.3.41' - knowmXchangeVersion = '5.0.13' langVersion = '3.11' logbackVersion = '1.1.11' loggingVersion = '1.2' @@ -72,7 +71,6 @@ configure(subprojects) { protocVersion = protobufVersion qrgenVersion = '1.3' slf4jVersion = '1.7.30' - springBootVersion = '2.5.6' os = osdetector.os == 'osx' ? 'mac' : osdetector.os == 'windows' ? 'win' : osdetector.os } @@ -98,7 +96,6 @@ configure([project(':cli'), project(':monitor'), project(':seednode'), project(':statsnode'), - project(':pricenode'), project(':apitest')]) { apply plugin: 'application' @@ -555,80 +552,6 @@ configure(project(':monitor')) { } -configure(project(':pricenode')) { - apply plugin: "org.springframework.boot" - - apply plugin: 'io.spring.dependency-management' - - mainClassName = 'bisq.price.Main' - - version = file("src/main/resources/version.txt").text.trim() - - jar.manifest.attributes( - "Implementation-Title": project.name, - "Implementation-Version": version) - - ext['log4j2.version'] = '2.17.0' - - dependencies { - implementation project(":common") - implementation project(":core") - annotationProcessor "org.projectlombok:lombok:$lombokVersion" - compileOnly "org.projectlombok:lombok:$lombokVersion" - implementation "com.google.code.gson:gson:$gsonVersion" - implementation "com.google.guava:guava:$guavaVersion" - implementation "commons-codec:commons-codec:$codecVersion" - implementation "org.apache.httpcomponents:httpcore:$httpcoreVersion" - implementation("org.apache.httpcomponents:httpclient:$httpclientVersion") { - exclude(module: 'commons-codec') - } - implementation("org.knowm.xchange:xchange-binance:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-bitbay:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-bitfinex:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-bitflyer:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-bitstamp:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-btcmarkets:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-cexio:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-coinbasepro:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-coinmarketcap:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-coinmate:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-coinone:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-exmo:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-hitbtc:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-huobi:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-independentreserve:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-kraken:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-luno:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-mercadobitcoin:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-paribu:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-poloniex:$knowmXchangeVersion") - implementation("org.knowm.xchange:xchange-quoine:$knowmXchangeVersion") - implementation("org.springframework.boot:spring-boot-starter-web:$springBootVersion") - testAnnotationProcessor "org.projectlombok:lombok:$lombokVersion" - testCompileOnly "org.projectlombok:lombok:$lombokVersion" - testImplementation "org.junit.jupiter:junit-jupiter-api:$jupiterVersion" - testImplementation "org.junit.jupiter:junit-jupiter-params:$jupiterVersion" - testImplementation "org.mockito:mockito-core:$mockitoVersion" - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$jupiterVersion") - } - - test { - useJUnitPlatform() - - // Disabled by default, since spot provider tests include connections to external API endpoints - // Can be enabled by adding -Dtest.pricenode.includeSpotProviderTests=true to the gradle command: - // ./gradlew test -Dtest.pricenode.includeSpotProviderTests=true - if (System.properties['test.pricenode.includeSpotProviderTests'] != 'true') { - project.logger.lifecycle('Pricenode: Skipping spot provider tests') - exclude 'bisq/price/spot/providers/**' - } - } - - task stage { - dependsOn assemble - } -} - configure(project(':seednode')) { apply plugin: 'com.github.johnrengelman.shadow' diff --git a/pricenode/Procfile b/pricenode/Procfile deleted file mode 100644 index ff15a5d03dc..00000000000 --- a/pricenode/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: if [ "$HIDDEN" == true ]; then ./tor/bin/run_tor java -jar -Dserver.port=$PORT build/libs/bisq-pricenode.jar; else java -jar -Dserver.port=$PORT build/libs/bisq-pricenode.jar; fi diff --git a/pricenode/README-HEROKU.md b/pricenode/README-HEROKU.md deleted file mode 100644 index ab6081cd96b..00000000000 --- a/pricenode/README-HEROKU.md +++ /dev/null @@ -1,42 +0,0 @@ -Deploy on Heroku --------- - -Run the following commands: - - heroku create - heroku buildpacks:add heroku/gradle - git push heroku master - curl https://your-app-123456.herokuapp.com/getAllMarketPrices - -To register the node as a Tor hidden service, first install the Heroku Tor buildpack: - - heroku buildpacks:add https://github.com/cbeams/heroku-buildpack-tor.git - git push heroku master - -> NOTE: this deployment will take a while, because the new buildpack must download and build Tor from source. - -Next, generate your Tor hidden service private key and .onion address: - - heroku run bash - ./tor/bin/tor -f torrc - -When the process reports that it is "100% bootstrapped", kill it, then copy the generated private key and .onion hostname values: - - cat build/tor-hidden-service/hostname - cat build/tor-hidden-service/private_key - exit - -> IMPORTANT: Save the private key value in a secure location so that this node can be re-created elsewhere with the same .onion address in the future if necessary. - -Now configure the hostname and private key values as environment variables for your Heroku app: - - heroku config:set HIDDEN=true HIDDEN_DOT_ONION=[your .onion] HIDDEN_PRIVATE_KEY="[your tor privkey]" - git push heroku master - -When the application finishes restarting, you should still be able to access it via the clearnet, e.g. with: - - curl https://your-app-123456.herokuapp.com/getAllMarketPrices - -And via your Tor Browser at: - - http://$YOUR_ONION/getAllMarketPrices diff --git a/pricenode/README.md b/pricenode/README.md deleted file mode 100644 index a39ad729a4b..00000000000 --- a/pricenode/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# bisq-pricenode - -## Overview - -The Bisq pricenode is a simple HTTP service that fetches, transforms and relays data from third-party price providers to Bisq exchange clients on request. Available prices include: - - - Bitcoin exchange rates, available at `/getAllMarketPrices`, and - - Bitcoin mining fee rates, available at `/getFees` - -Pricenodes are deployed in production as Tor hidden services. This is not because the location of these nodes needs to be kept secret, but rather so that Bisq exchange clients do not need to exit the Tor network in order to get price data. - -Anyone can run a pricenode, but it must be _discoverable_ in order for it to do any good. For exchange clients to discover your pricenode, its .onion address must be hard-coded in the Bisq exchange client's `ProvidersRepository` class. Alternatively, users can point explicitly to given pricenode (or set of pricenodes) with the exchange client's `--providers` command line option. - -Pricenodes can be deployed anywhere Java and Tor binaries can be run. Instructions below cover deployment on localhost, and instructions [how to deploy on Heroku](README-HEROKU.md) are also available. - -Pricenodes should be cheap to run with regard to both time and money. The application itself is non-resource intensive and can be run on the low-end of most providers' paid tiers. - -A [pricenode operator](https://github.com/bisq-network/roles/issues/5)'s main responsibilities are to ensure their node(s) are available and up-to-date. Releases are currently source-only, with the assumption that most operators will favor Git-based "push to deploy" workflows. To stay up to date with releases, operators can [subscribe to this repository's releases.atom feed](https://github.com/bisq-network/pricenode/releases.atom) and/or get notifications in the `#pricenode` Slack channel. - -Operating a production pricenode is a valuable service to the Bisq network, and operators should issue BSQ compensation requests accordingly. - - -## Prerequisites for running a pricenode - -To run a pricenode, you will need: - - - JDK 8 if you want to build and run a node locally. - - The `tor` binary (e.g. `brew install tor`) if you want to run a hidden service locally. - -## How to deploy for production - -### Install - -Run the one-command installer: - -```bash -curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/install_pricenode_debian.sh | sudo bash -``` - -At the end of the installer script, it should print your Tor onion hostname. - -### Test - -To manually test endpoints, run each of the following: - -``` bash -curl http://localhost:8080/getAllMarketPrices -curl http://localhost:8080/getFees -curl http://localhost:8080/getParams -curl http://localhost:8080/info -``` - -### Monitoring - -If you run a main pricenode, you also are obliged to activate the monitoring feed by running - -```bash -bash <(curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/monitor/install_collectd_debian.sh) -``` -Follow the instruction given by the script and report your certificate to the [@bisq-network/monitoring](https://github.com/orgs/bisq-network/teams/monitoring-operators) team. - -Furthermore, you are obliged to provide network size data to the monitor by running -```bash -curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/install_networksize_debian.sh | sudo bash -``` - -### Updating - -Update your bisq code in /bisq/bisq with ```git pull``` - -Then build an updated pricenode: -```./gradlew :pricenode:installDist -x test``` - -## How to deploy elsewhere - - - [README-HEROKU.md](README-HEROKU.md) - - [docker/README.md](docker/README.md) - - -## Bitcoin mining fee estimates - -The pricenode exposes a service API to Bisq clients under `/getFees`. - -This API returns a mining fee rate estimate, representing an average of several mining fee rate values retrieved from different `mempool.space` instances. - -To configure which `mempool.space` instances are queried to calculate this average, see the relevant section in the file `application.properties`. diff --git a/pricenode/TODO.md b/pricenode/TODO.md deleted file mode 100644 index 6ce8d074a83..00000000000 --- a/pricenode/TODO.md +++ /dev/null @@ -1,20 +0,0 @@ -# Refactorings - -The list of stuff remaining to complete the PR at https://github.com/bisq-network/pricenode/pull/7 - - - Document provider implementations w/ links to API docs, etc - - Add integration tests - - Document / discuss how operators should (ideally) operate their pricenodes on a push-to-deploy model, e.g. how it's done on Heroku - -## Non-refactorings - -Most or all of these will become individual issues / PRs. Just capturing them here for convenience now. Not all may make sense. - - - Deprecate existing get* endpoints (e.g. /getAllMarketPrices) in favor of '/exchange-rates', '/fee-estimate; - - Eliminate dependency on bisq-core (only real need now is CurrencyUtil for list of supported coins) - - Remove command line args for fee estimation params; hard-code these values and update them via commits, not via one-off changes by each operator - - Remove 'getParams' in favor of Boot actuator endpoint - - Update bisq-network/exchange to refer to 'provider' as 'pricenode' - - Invert the dependency arrangement. Move 'ProviderRepository' et al from bisq-network/exchange here into - bisq-network/pricenode and have bisq-network/exchange depend on it as a client lib - - Save bandwidth and be idiomatic by not pretty-printing json returned from /getAllMarketPrices et al diff --git a/pricenode/bisq-pricenode.env b/pricenode/bisq-pricenode.env deleted file mode 100644 index f77a6675c47..00000000000 --- a/pricenode/bisq-pricenode.env +++ /dev/null @@ -1 +0,0 @@ -JAVA_OPTS="-XX:+ExitOnOutOfMemoryError" diff --git a/pricenode/bisq-pricenode.service b/pricenode/bisq-pricenode.service deleted file mode 100644 index 8123387228b..00000000000 --- a/pricenode/bisq-pricenode.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Bisq Price Node -After=network.target - -[Service] -SyslogIdentifier=bisq-pricenode -EnvironmentFile=/etc/default/bisq-pricenode.env -ExecStart=/bisq/bisq/bisq-pricenode 2 -ExecStop=/bin/kill -TERM ${MAINPID} -Restart=on-failure - -User=bisq -Group=bisq - -PrivateTmp=true -ProtectSystem=full -NoNewPrivileges=true -PrivateDevices=true -MemoryDenyWriteExecute=false - -[Install] -WantedBy=multi-user.target diff --git a/pricenode/collectd.conf.snippet b/pricenode/collectd.conf.snippet deleted file mode 100644 index 49d93c5eda3..00000000000 --- a/pricenode/collectd.conf.snippet +++ /dev/null @@ -1,5 +0,0 @@ -LoadPlugin exec - - - Exec "__USER_GROUP__" "__SCRAPERSCRIPT__" - diff --git a/pricenode/docker/Dockerfile b/pricenode/docker/Dockerfile deleted file mode 100644 index cc195cd63b5..00000000000 --- a/pricenode/docker/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### -# The directory of the Dockerfile should contain your 'hostname' and 'private_key' files. -# In the docker-compose.yml file you can pass the ONION_ADDRESS referenced below. -### - -# pull base image -FROM openjdk:8-jdk - -RUN apt-get update && apt-get install -y --no-install-recommends \ - vim \ - tor \ - fakeroot \ - sudo \ - openjfx && rm -rf /var/lib/apt/lists/* - -RUN git clone https://github.com/bisq-network/pricenode.git -WORKDIR /pricenode/ -RUN ./gradlew assemble - -COPY loop.sh start_node.sh start_tor.sh ./ -COPY hostname private_key /var/lib/tor/ -COPY torrc /etc/tor/ -RUN chmod +x *.sh && chown debian-tor:debian-tor /etc/tor/torrc /var/lib/tor/hostname /var/lib/tor/private_key - -CMD ./start_tor.sh && ./start_node.sh -#CMD tail -f /dev/null diff --git a/pricenode/docker/README.md b/pricenode/docker/README.md deleted file mode 100644 index 5f7a4e6d11f..00000000000 --- a/pricenode/docker/README.md +++ /dev/null @@ -1,43 +0,0 @@ -Needed information to start a pricenode -== - -Copy to this directory: --- - -* a tor `hostname` file, containing your onion address -* a tor `private_key` file, containing the private key for your tor hidden service - -Edit docker-compose.yml: --- - -* fill in your public and private api keys (needs a btcaverage developer subscription) - -Needed software to start a pricenode -== - -* docker -* docker-compose - -How to start -== - -`docker-compose up -d` - - -How to monitor -== - -See if it's running: `docker ps` - -Check the logs: `docker-compose logs` - - -Notes when using CoreOs -== - -Using CoreOs as host OS is entirely optional! - -* the cloudconfig.yml file is a configuration file for starting a coreos machine -from scratch. -* when installing a Coreos server, docker-compose needs to be additionally installed next to the -already provided docker installation diff --git a/pricenode/docker/cloudconfig.yml b/pricenode/docker/cloudconfig.yml deleted file mode 100644 index d6450b70f79..00000000000 --- a/pricenode/docker/cloudconfig.yml +++ /dev/null @@ -1,103 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: off - units: - - name: iptables-restore.service - enable: true - command: start - - name: create-swap.service - command: start - runtime: true - content: | - [Unit] - Description=Create swap file - Before=swap.service - - [Service] - Type=oneshot - Environment="SWAPFILE=/2GiB.swap" - ExecStart=/usr/bin/touch ${SWAPFILE} - ExecStart=/usr/bin/chattr +C ${SWAPFILE} - ExecStart=/usr/bin/fallocate -l 2048m ${SWAPFILE} - ExecStart=/usr/bin/chmod 600 ${SWAPFILE} - ExecStart=/usr/sbin/mkswap ${SWAPFILE} - - [Install] - WantedBy=multi-user.target - - name: swap.service - command: start - content: | - [Unit] - Description=Turn on swap - - [Service] - Type=oneshot - Environment="SWAPFILE=/2GiB.swap" - RemainAfterExit=true - ExecStartPre=/usr/sbin/losetup -f ${SWAPFILE} - ExecStart=/usr/bin/sh -c "/sbin/swapon $(/usr/sbin/losetup -j ${SWAPFILE} | /usr/bin/cut -d : -f 1)" - ExecStop=/usr/bin/sh -c "/sbin/swapoff $(/usr/sbin/losetup -j ${SWAPFILE} | /usr/bin/cut -d : -f 1)" - ExecStopPost=/usr/bin/sh -c "/usr/sbin/losetup -d $(/usr/sbin/losetup -j ${SWAPFILE} | /usr/bin/cut -d : -f 1)" - - [Install] - WantedBy=multi-user.target - - name: restart.service - content: | - [Unit] - Description=Restart docker containers - - [Service] - Type=oneshot - ExecStart=/home/core/docker/restartContainers.sh - - name: restart.timer - command: start - content: | - [Unit] - Description=Restarts the app container 2 times a week - - [Timer] - OnCalendar=Mon,Thu *-*-* 6:0:0 - -write_files: - - path: /etc/sysctl.d/swap.conf - permissions: 0644 - owner: root - content: | - vm.swappiness=10 - vm.vfs_cache_pressure=50 - -write_files: - - path: /etc/ssh/sshd_config - permissions: 0600 - owner: root - content: | - # Use most defaults for sshd configuration. - UsePrivilegeSeparation sandbox - Subsystem sftp internal-sftp - UseDNS no - - PermitRootLogin no - AllowUsers core - AuthenticationMethods publickey - -write_files: - - path: /var/lib/iptables/rules-save - permissions: 0644 - owner: 'root:root' - content: | - *filter - :INPUT DROP [0:0] - :FORWARD DROP [0:0] - :OUTPUT ACCEPT [0:0] - -A INPUT -i lo -j ACCEPT - -A INPUT -i eth1 -j ACCEPT - -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - -A INPUT -p tcp -m tcp --dport 22 -j ACCEPT - -A INPUT -p tcp -m tcp --dport 80 -j ACCEPT - -A INPUT -p icmp -m icmp --icmp-type 0 -j ACCEPT - -A INPUT -p icmp -m icmp --icmp-type 3 -j ACCEPT - -A INPUT -p icmp -m icmp --icmp-type 11 -j ACCEPT - COMMIT - # the last line of the file needs to be a blank line or a comment diff --git a/pricenode/docker/docker-compose.yml b/pricenode/docker/docker-compose.yml deleted file mode 100644 index 9ecdb846884..00000000000 --- a/pricenode/docker/docker-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: '3' - -# Fill in your own BTCAVERAGE public and private keys - -services: - pricenode: - restart: unless-stopped - build: - context: . - image: bisq:pricenode - ports: - - 80:80 - - 8080:8080 - environment: - - BTCAVERAGE_PRIVKEY=!!!!!!!!!!!!!!!!!!!!!!!!! YOUR PRIVATE KEY !!!!!!!!!!!!!!!!!!!!!!!!!!! - - BTCAVERAGE_PUBKEY=!!!!!!!!!!!!!!!!!!!!!!!!!! YOUR PUBKEY !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - entropy: - restart: always - image: harbur/haveged:1.7c-1 - container_name: haveged-entropy - privileged: true diff --git a/pricenode/docker/installDockerCompose.sh b/pricenode/docker/installDockerCompose.sh deleted file mode 100644 index f41ef503076..00000000000 --- a/pricenode/docker/installDockerCompose.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -mkdir -p /opt/bin -curl -L `curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(contains("Linux") and contains("x86_64"))'` > /opt/bin/docker-compose -chmod +x /opt/bin/docker-compose diff --git a/pricenode/docker/loop.sh b/pricenode/docker/loop.sh deleted file mode 100644 index 1382fbb13c5..00000000000 --- a/pricenode/docker/loop.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -while true -do -echo `date` "(Re)-starting node" -java -jar ./build/libs/bisq-pricenode.jar 2 2 -echo `date` "node terminated unexpectedly!!" -sleep 3 -done diff --git a/pricenode/docker/rebuildAndRestart.sh b/pricenode/docker/rebuildAndRestart.sh deleted file mode 100755 index a30d2b21db9..00000000000 --- a/pricenode/docker/rebuildAndRestart.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -docker-compose build --no-cache && docker-compose up -d -docker image prune -f -docker-compose logs -f diff --git a/pricenode/docker/start_node.sh b/pricenode/docker/start_node.sh deleted file mode 100644 index 225994130a1..00000000000 --- a/pricenode/docker/start_node.sh +++ /dev/null @@ -1 +0,0 @@ -nohup sh loop.sh diff --git a/pricenode/docker/start_tor.sh b/pricenode/docker/start_tor.sh deleted file mode 100644 index 40fa1e9e0ce..00000000000 --- a/pricenode/docker/start_tor.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -# sudo -u debian-tor -nohup sudo -u debian-tor tor > /dev/null 2>errors_tor.log & diff --git a/pricenode/docker/torrc b/pricenode/docker/torrc deleted file mode 100644 index 9de1a9166e2..00000000000 --- a/pricenode/docker/torrc +++ /dev/null @@ -1,2 +0,0 @@ -HiddenServiceDir /var/lib/tor/ -HiddenServicePort 80 127.0.0.1:8080 diff --git a/pricenode/install_hsversion_debian.sh b/pricenode/install_hsversion_debian.sh deleted file mode 100755 index 631a02787cf..00000000000 --- a/pricenode/install_hsversion_debian.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -set -e - -echo "[*] Network Size Monitoring installation script" - -##### change paths if necessary for your system - -ROOT_USER=root - -SCRAPER_HOME=/journalreader -SCRAPER_USER=journalreader -SCRAPER_GROUP=systemd-journal - -##### -echo "[*] Checking environment..." -if [ ! -f "/etc/collectd/collectd.conf" ]; then - echo 'Collectd is not installed. Did you do the install_monitoring_debian.sh?' - echo 'Exiting...' - exit -fi -if ! grep -q "journalreader" /etc/passwd; then - echo 'User not found. Did you run the install_networksize_debian.sh?' - echo 'Exiting...' - exit -fi - -echo "[*] Installing journal parser script" -curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/journalscraper_hsversion.sh > /tmp/journalscraper_hsversion.sh -sudo -H -i -u "${ROOT_USER}" install -c -o "${SCRAPER_USER}" -g "${SCRAPER_GROUP}" -m 744 /tmp/journalscraper_hsversion.sh "${SCRAPER_HOME}/scraperscript_hsversion.sh" - -echo "[*] Installing collectd config" -curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/collectd.conf.snippet > /tmp/collectd.conf.snippet -sudo -H -i -u "${ROOT_USER}" sed -i -e "s/LoadPlugin exec//" /tmp/collectd.conf.snippet -sudo -H -i -u "${ROOT_USER}" /bin/sh -c "cat /tmp/collectd.conf.snippet >> /etc/collectd/collectd.conf" -sudo -H -i -u "${ROOT_USER}" sed -i -e "s/__USER_GROUP__/${SCRAPER_USER}:${SCRAPER_GROUP}/" /etc/collectd/collectd.conf -sudo -H -i -u "${ROOT_USER}" sed -i -e "s!__SCRAPERSCRIPT__!${SCRAPER_HOME}/scraperscript_hsversion.sh!" /etc/collectd/collectd.conf - -echo "[*] Restarting services" -sudo -H -i -u "${ROOT_USER}" systemctl restart collectd.service - -echo '[*] Done!' diff --git a/pricenode/install_networksize_debian.sh b/pricenode/install_networksize_debian.sh deleted file mode 100755 index bc25b3c6ddd..00000000000 --- a/pricenode/install_networksize_debian.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -set -e - -echo "[*] Network Size Monitoring installation script" - -##### change paths if necessary for your system - -ROOT_USER=root - -SCRAPER_HOME=/journalreader -SCRAPER_USER=journalreader -SCRAPER_GROUP=systemd-journal - -##### -echo "[*] Checking environment..." -if [ ! -f "/etc/collectd/collectd.conf" ]; then - echo 'Collectd is not installed. Did you do the install_monitoring_debian.sh?' - echo 'Exiting...' - exit -fi - -echo "[*] Creating journal reader user" -sudo -H -i -u "${ROOT_USER}" useradd -d "${SCRAPER_HOME}" -G "${SCRAPER_GROUP}" "${SCRAPER_USER}" -sudo -H -i -u "${ROOT_USER}" mkdir -p "${SCRAPER_HOME}" -sudo -H -i -u "${ROOT_USER}" chown "${SCRAPER_USER}":"${SCRAPER_GROUP}" ${SCRAPER_HOME} - -echo "[*] Installing journal parser script" -curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/journalscraper.sh > /tmp/journalscraper.sh -sudo -H -i -u "${ROOT_USER}" install -c -o "${SCRAPER_USER}" -g "${SCRAPER_GROUP}" -m 744 /tmp/journalscraper.sh "${SCRAPER_HOME}/scraperscript.sh" - -echo "[*] Installing collectd config" -curl -s https://raw.githubusercontent.com/bisq-network/bisq/master/pricenode/collectd.conf.snippet > /tmp/collectd.conf.snippet -sudo -H -i -u "${ROOT_USER}" /bin/sh -c "cat /tmp/collectd.conf.snippet >> /etc/collectd/collectd.conf" -sudo -H -i -u "${ROOT_USER}" sed -i -e "s/__USER_GROUP__/${SCRAPER_USER}:${SCRAPER_GROUP}/" /etc/collectd/collectd.conf -sudo -H -i -u "${ROOT_USER}" sed -i -e "s!__SCRAPERSCRIPT__!${SCRAPER_HOME}/scraperscript.sh!" /etc/collectd/collectd.conf - -sudo -H -i -u "${ROOT_USER}" systemctl enable collectd.service - -echo "[*] Restarting services" -sudo -H -i -u "${ROOT_USER}" systemctl restart collectd.service - -echo '[*] Done!' diff --git a/pricenode/install_pricenode_debian.sh b/pricenode/install_pricenode_debian.sh deleted file mode 100755 index 1f4b7e2abdf..00000000000 --- a/pricenode/install_pricenode_debian.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -set -e - -echo "[*] Bisq bisq-pricenode installation script" - -##### change as necessary for your system - -SYSTEMD_SERVICE_HOME=/etc/systemd/system -SYSTEMD_ENV_HOME=/etc/default - -ROOT_USER=root -ROOT_GROUP=root -#ROOT_HOME=/root - -BISQ_USER=bisq -BISQ_GROUP=bisq -BISQ_HOME=/bisq - -BISQ_REPO_URL=https://github.com/bisq-network/bisq -BISQ_REPO_NAME=bisq -BISQ_REPO_TAG=master -BISQ_LATEST_RELEASE=master -BISQ_TORHS=pricenode - -TOR_PKG="tor" -#TOR_USER=debian-tor -TOR_GROUP=debian-tor -TOR_CONF=/etc/tor/torrc -TOR_RESOURCES=/var/lib/tor - -##### - -echo "[*] Upgrading apt packages" -sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get update -q -sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get upgrade -qq -y - -echo "[*] Installing Tor" -sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get install -qq -y "${TOR_PKG}" - -echo "[*] Adding Tor configuration" -if ! grep "${BISQ_TORHS}" /etc/tor/torrc >/dev/null 2>&1;then - sudo -H -i -u "${ROOT_USER}" sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${BISQ_TORHS}/ >> ${TOR_CONF}" - sudo -H -i -u "${ROOT_USER}" sh -c "echo HiddenServicePort 80 127.0.0.1:8080 >> ${TOR_CONF}" - sudo -H -i -u "${ROOT_USER}" sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONF}" -fi - -echo "[*] Creating Bisq user with Tor access" -sudo -H -i -u "${ROOT_USER}" useradd -d "${BISQ_HOME}" -G "${TOR_GROUP}" "${BISQ_USER}" - -echo "[*] Creating Bisq homedir" -sudo -H -i -u "${ROOT_USER}" mkdir -p "${BISQ_HOME}" -sudo -H -i -u "${ROOT_USER}" chown "${BISQ_USER}":"${BISQ_GROUP}" ${BISQ_HOME} - -echo "[*] Cloning Bisq repo" -sudo -H -i -u "${BISQ_USER}" git config --global advice.detachedHead false -sudo -H -i -u "${BISQ_USER}" git clone --branch "${BISQ_REPO_TAG}" "${BISQ_REPO_URL}" "${BISQ_HOME}/${BISQ_REPO_NAME}" - -echo "[*] Installing OpenJDK 11" -sudo -H -i -u "${ROOT_USER}" apt-get install -qq -y openjdk-11-jdk - -echo "[*] Checking out Bisq ${BISQ_LATEST_RELEASE}" -sudo -H -i -u "${BISQ_USER}" sh -c "cd ${BISQ_HOME}/${BISQ_REPO_NAME} && git checkout ${BISQ_LATEST_RELEASE}" - -echo "[*] Building Bisq from source" -sudo -H -i -u "${BISQ_USER}" sh -c "cd ${BISQ_HOME}/${BISQ_REPO_NAME} && ./gradlew :pricenode:installDist -x test < /dev/null" # redirect from /dev/null is necessary to workaround gradlew non-interactive shell hanging issue - -echo "[*] Installing bisq-pricenode systemd service" -sudo -H -i -u "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${BISQ_HOME}/${BISQ_REPO_NAME}/pricenode/bisq-pricenode.service" "${SYSTEMD_SERVICE_HOME}" -sudo -H -i -u "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${BISQ_HOME}/${BISQ_REPO_NAME}/pricenode/bisq-pricenode.env" "${SYSTEMD_ENV_HOME}" - -echo "[*] Reloading systemd daemon configuration" -sudo -H -i -u "${ROOT_USER}" systemctl daemon-reload - -echo "[*] Enabling bisq-pricenode service" -sudo -H -i -u "${ROOT_USER}" systemctl enable bisq-pricenode.service - -echo "[*] Starting bisq-pricenode service" -sudo -H -i -u "${ROOT_USER}" systemctl start bisq-pricenode.service -sleep 5 -sudo -H -i -u "${ROOT_USER}" journalctl --no-pager --unit bisq-pricenode - -echo "[*] Restarting Tor" -sudo -H -i -u "${ROOT_USER}" service tor restart -sleep 5 - -echo '[*] Done!' -echo -n '[*] Access your pricenode at http://' -cat "${TOR_RESOURCES}/${BISQ_TORHS}/hostname" - -exit 0 diff --git a/pricenode/journalscraper.sh b/pricenode/journalscraper.sh deleted file mode 100755 index df80ddc3ada..00000000000 --- a/pricenode/journalscraper.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -HOSTNAME="${COLLECTD_HOSTNAME:-localhost}" -INTERVAL=750 - -last=$(date +"%F %T" -d "$INTERVAL seconds ago") -while true; -do - now=$(date +"%F %T") - - journalctl -u bisq-pricenode --since="$last" --until="$now" | grep -Eo "getAllMarketPrices.*bisq/[0-9].[0-9].[0-9]" | cut -d / -f 2 | sort | uniq -c | while read -r line; do - number=$(echo "${line}" | cut -d ' ' -f 1); - version=$(echo "${line}" | cut -d \ -f 2); - version=${version//./_}; - echo "PUTVAL $HOSTNAME/requestsPer750Seconds/gauge-v$version interval=$INTERVAL N:$number"; - done - last=$now - - sleep $INTERVAL -done diff --git a/pricenode/journalscraper_hsversion.sh b/pricenode/journalscraper_hsversion.sh deleted file mode 100755 index f5d710b8b70..00000000000 --- a/pricenode/journalscraper_hsversion.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -HOSTNAME="${COLLECTD_HOSTNAME:-localhost}" -INTERVAL=750 - -last=$(date +"%F %T" -d "$INTERVAL seconds ago") -while true; -do - now=$(date +"%F %T") - - journalctl -u bisq-pricenode --since="$last" --until="$now" | grep -Eo "getAllMarketPrices.*HSv[0-9]" | grep -o "HSv[0-9]" | sort | uniq -c | while read -r line; do - number=$(echo "${line}" | cut -d ' ' -f 1); - version=$(echo "${line}" | cut -d \ -f 2); - version=${version//./_}; - echo "PUTVAL $HOSTNAME/hsversionStats/gauge-$version interval=$INTERVAL N:$number"; - done - last=$now - - sleep $INTERVAL -done diff --git a/pricenode/src/main/java/bisq/price/Main.java b/pricenode/src/main/java/bisq/price/Main.java deleted file mode 100644 index 0bb12678d27..00000000000 --- a/pricenode/src/main/java/bisq/price/Main.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price; - -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.UncheckedIOException; - -import java.util.Properties; - -@SpringBootApplication -public class Main { - - public static void main(String[] args) { - new SpringApplicationBuilder(Main.class) - .properties(bisqProperties()) - .run(args); - } - - private static Properties bisqProperties() { - Properties props = new Properties(); - File propsFile = new File(System.getenv("HOME"), ".config/bisq.properties"); - if (propsFile.exists()) { - try { - props.load(new FileInputStream(propsFile)); - } catch (IOException ex) { - throw new UncheckedIOException(ex); - } - } - return props; - } -} diff --git a/pricenode/src/main/java/bisq/price/PriceController.java b/pricenode/src/main/java/bisq/price/PriceController.java deleted file mode 100644 index d3c98027de8..00000000000 --- a/pricenode/src/main/java/bisq/price/PriceController.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price; - -import org.springframework.web.bind.annotation.ModelAttribute; - -import javax.servlet.http.HttpServletRequest; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class PriceController { - - protected final Logger log = LoggerFactory.getLogger(this.getClass()); - - @ModelAttribute - public void logRequest(HttpServletRequest request) { - log.info("Incoming {} request from: {}", request.getServletPath(), request.getHeader("User-Agent")); - } -} diff --git a/pricenode/src/main/java/bisq/price/PriceProvider.java b/pricenode/src/main/java/bisq/price/PriceProvider.java deleted file mode 100644 index cbeefd9422b..00000000000 --- a/pricenode/src/main/java/bisq/price/PriceProvider.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price; - -import bisq.common.UserThread; - -import org.springframework.context.SmartLifecycle; - -import java.time.Duration; - -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class PriceProvider implements SmartLifecycle, Supplier { - - protected final Logger log = LoggerFactory.getLogger(this.getClass()); - - private final Timer timer = new Timer(true); - - protected final Duration refreshInterval; - - private T cachedResult; - - public PriceProvider(Duration refreshInterval) { - this.refreshInterval = refreshInterval; - log.info("will refresh every {}", refreshInterval); - } - - @Override - public final T get() { - return cachedResult; - } - - @Override - public final void start() { - // do the initial refresh asynchronously - UserThread.runAfter(() -> { - try { - refresh(); - } catch (Throwable t) { - log.warn("initial refresh failed", t); - } - }, 1, TimeUnit.MILLISECONDS); - - timer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - try { - refresh(); - } catch (Throwable t) { - // we only log scheduled calls to refresh that fail to ensure that - // the application does *not* halt, assuming the failure is temporary - // and on the side of the upstream price provider, eg. BitcoinAverage - log.warn("refresh failed", t); - } - } - }, refreshInterval.toMillis(), refreshInterval.toMillis()); - } - - private void refresh() { - long ts = System.currentTimeMillis(); - - cachedResult = doGet(); - - log.info("refresh took {} ms.", (System.currentTimeMillis() - ts)); - - onRefresh(); - } - - protected abstract T doGet(); - - protected void onRefresh() { - } - - @Override - public void stop() { - timer.cancel(); - } - - @Override - public void stop(Runnable callback) { - stop(); - callback.run(); - } - - @Override - public boolean isAutoStartup() { - return true; - } - - @Override - public boolean isRunning() { - return cachedResult != null; - } - - @Override - public int getPhase() { - return 0; - } -} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRate.java b/pricenode/src/main/java/bisq/price/mining/FeeRate.java deleted file mode 100644 index b5c25bcfecf..00000000000 --- a/pricenode/src/main/java/bisq/price/mining/FeeRate.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.mining; - -/** - * A value object representing the mining fee rate for a given base currency. - */ -public class FeeRate { - - private final String currency; - private final long price; - private final long minimumFee; - private final long timestamp; - - public FeeRate(String currency, long price, long minimumFee, long timestamp) { - this.currency = currency; - this.price = price; - this.minimumFee = minimumFee; - this.timestamp = timestamp; - } - - public String getCurrency() { - return currency; - } - - public long getPrice() { - return price; - } - - public long getMinimumFee() { - return minimumFee; - } - - public long getTimestamp() { - return timestamp; - } -} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRateController.java b/pricenode/src/main/java/bisq/price/mining/FeeRateController.java deleted file mode 100644 index b0ac4710b5c..00000000000 --- a/pricenode/src/main/java/bisq/price/mining/FeeRateController.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.mining; - -import bisq.price.PriceController; - -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Map; - -@RestController -class FeeRateController extends PriceController { - - private final FeeRateService feeRateService; - - public FeeRateController(FeeRateService feeRateService) { - this.feeRateService = feeRateService; - } - - @GetMapping(path = "/getFees") - public Map getFees() { - return feeRateService.getFees(); - } -} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/FeeRateProvider.java deleted file mode 100644 index 4507a992f6c..00000000000 --- a/pricenode/src/main/java/bisq/price/mining/FeeRateProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.mining; - -import bisq.price.PriceProvider; - -import java.time.Duration; - -/** - * Abstract base class for providers of mining {@link FeeRate} data. - */ -public abstract class FeeRateProvider extends PriceProvider { - - public static final long MIN_FEE_RATE_FOR_WITHDRAWAL = 1; // satoshi/vbyte - public static final long MIN_FEE_RATE_FOR_TRADING = 10; // satoshi/vbyte - public static final long MAX_FEE_RATE = 1000; - - public FeeRateProvider(Duration refreshInterval) { - super(refreshInterval); - } -} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRateService.java b/pricenode/src/main/java/bisq/price/mining/FeeRateService.java deleted file mode 100644 index ec86c93892d..00000000000 --- a/pricenode/src/main/java/bisq/price/mining/FeeRateService.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.mining; - -import bisq.common.config.Config; - -import org.springframework.stereotype.Service; - -import java.time.Instant; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * High-level mining {@link FeeRate} operations. - */ -@Service -public class FeeRateService { - - private final List providers; - protected final Logger log = LoggerFactory.getLogger(this.getClass()); - - /** - * Construct a {@link FeeRateService} with a list of all {@link FeeRateProvider} - * implementations discovered via classpath scanning. - * - * @param providers all {@link FeeRateProvider} implementations in ascending - * order of precedence - */ - public FeeRateService(List providers) { - this.providers = providers; - } - - public Map getFees() { - Map metadata = new HashMap<>(); - Map allFeeRates = new HashMap<>(); - - AtomicLong sumOfAllFeeRates = new AtomicLong(); - AtomicLong sumOfAllMinFeeRates = new AtomicLong(); - AtomicInteger amountOfFeeRates = new AtomicInteger(); - - // Process each provider, retrieve and store their fee rate - providers.forEach(p -> { - FeeRate feeRate = p.get(); - if (feeRate == null) { - log.warn("feeRate is null, provider={} ", p.toString()); - return; - } - String currency = feeRate.getCurrency(); - if ("BTC".equals(currency)) { - sumOfAllFeeRates.getAndAdd(feeRate.getPrice()); - sumOfAllMinFeeRates.getAndAdd(feeRate.getMinimumFee()); - amountOfFeeRates.getAndAdd(1); - } - }); - - // Calculate the average - long averageFeeRate = (amountOfFeeRates.intValue() > 0) - ? sumOfAllFeeRates.longValue() / amountOfFeeRates.intValue() - : FeeRateProvider.MIN_FEE_RATE_FOR_TRADING; - long averageMinFeeRate = (amountOfFeeRates.intValue() > 0) - ? sumOfAllMinFeeRates.longValue() / amountOfFeeRates.intValue() - : FeeRateProvider.MIN_FEE_RATE_FOR_WITHDRAWAL; - - // Make sure the returned value is within the min-max range - averageFeeRate = Math.max(averageFeeRate, FeeRateProvider.MIN_FEE_RATE_FOR_TRADING); - averageFeeRate = Math.min(averageFeeRate, FeeRateProvider.MAX_FEE_RATE); - averageMinFeeRate = Math.max(averageMinFeeRate, FeeRateProvider.MIN_FEE_RATE_FOR_WITHDRAWAL); - averageMinFeeRate = Math.min(averageMinFeeRate, FeeRateProvider.MAX_FEE_RATE); - - // Prepare response: Add timestamp of now - // Since this is an average, the timestamp is associated with when the moment in - // time when the avg was computed - metadata.put(Config.BTC_FEES_TS, Instant.now().getEpochSecond()); - - // Prepare response: Add the fee average - allFeeRates.put(Config.BTC_TX_FEE, averageFeeRate); - allFeeRates.put(Config.BTC_MIN_TX_FEE, averageMinFeeRate); - - // Build response - return new HashMap<>() {{ - putAll(metadata); - put(Config.LEGACY_FEE_DATAMAP, allFeeRates); - }}; - } -} diff --git a/pricenode/src/main/java/bisq/price/mining/providers/MempoolFeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/providers/MempoolFeeRateProvider.java deleted file mode 100644 index 71fbfefc37d..00000000000 --- a/pricenode/src/main/java/bisq/price/mining/providers/MempoolFeeRateProvider.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.mining.providers; - -import bisq.price.PriceController; -import bisq.price.mining.FeeRate; -import bisq.price.mining.FeeRateProvider; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Primary; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.core.annotation.Order; -import org.springframework.core.env.CommandLinePropertySource; -import org.springframework.core.env.Environment; -import org.springframework.http.RequestEntity; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -import java.time.Duration; -import java.time.Instant; - -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** - * {@link FeeRateProvider} that interprets the Mempool API format to retrieve a mining - * fee estimate. See https://mempool.space. - */ -abstract class MempoolFeeRateProvider extends FeeRateProvider { - - private static final int DEFAULT_MAX_BLOCKS = 2; - private static final int DEFAULT_REFRESH_INTERVAL = 2; - - // Keys of properties defining the available Mempool API endpoints. To enable them, - // simply uncomment and adjust the corresponding lines in application.properties - private static final String MEMPOOL_HOSTNAME_KEY_1 = "bisq.price.mining.providers.mempoolHostname.1"; - private static final String MEMPOOL_HOSTNAME_KEY_2 = "bisq.price.mining.providers.mempoolHostname.2"; - private static final String MEMPOOL_HOSTNAME_KEY_3 = "bisq.price.mining.providers.mempoolHostname.3"; - private static final String MEMPOOL_HOSTNAME_KEY_4 = "bisq.price.mining.providers.mempoolHostname.4"; - private static final String MEMPOOL_HOSTNAME_KEY_5 = "bisq.price.mining.providers.mempoolHostname.5"; - - private static final RestTemplate restTemplate = new RestTemplate(); - - // TODO: As of the switch to the mempool.space API this field and related members are - // now dead code and should be removed, including removing the positional - // command-line argument from startup scripts. Operators need to be notified of this - // when it happens. - private final int maxBlocks; - - protected Environment env; - - public MempoolFeeRateProvider(Environment env) { - super(Duration.ofMinutes(refreshInterval(env))); - this.env = env; - this.maxBlocks = maxBlocks(env); - } - - protected FeeRate doGet() { - // Default value is the minimum rate. If the connection to the fee estimate - // provider fails, we fall back to this value. - try { - return getEstimatedFeeRate(); - } - catch (Exception e) { - // Something happened with the connection - log.error("Error retrieving bitcoin mining fee estimation: " + e.getMessage()); - } - - return new FeeRate("BTC", MIN_FEE_RATE_FOR_TRADING, MIN_FEE_RATE_FOR_WITHDRAWAL, Instant.now().getEpochSecond()); - } - - private FeeRate getEstimatedFeeRate() { - Set> feeRatePredictions = getFeeRatePredictions(); - long estimatedFeeRate = feeRatePredictions.stream() - .filter(p -> p.getKey().equalsIgnoreCase("halfHourFee")) - .map(Map.Entry::getValue) - .findFirst() - .map(r -> Math.max(r, MIN_FEE_RATE_FOR_TRADING)) - .map(r -> Math.min(r, MAX_FEE_RATE)) - .orElse(MIN_FEE_RATE_FOR_TRADING); - long economyFee = feeRatePredictions.stream() - .filter(p -> p.getKey().equalsIgnoreCase("economyFee")) - .map(Map.Entry::getValue) - .findFirst() - .orElse(MIN_FEE_RATE_FOR_WITHDRAWAL); - log.info("Retrieved estimated mining fee of {} sat/vB and economyFee of {} sat/vB from {}", estimatedFeeRate, economyFee, getMempoolApiHostname()); - return new FeeRate("BTC", estimatedFeeRate, economyFee, Instant.now().getEpochSecond()); - } - - private Set> getFeeRatePredictions() { - return restTemplate.exchange( - RequestEntity - .get(UriComponentsBuilder - // See https://github.com/bisq-network/projects/issues/27 - .fromUriString("https://" + getMempoolApiHostname() + "/api/v1/fees/recommended") - .build().toUri()) - .build(), - new ParameterizedTypeReference>() { } - ).getBody().entrySet(); - } - - /** - * Return the hostname of the fee estimation API endpoint. No prefix (https://), no - * suffix (trailing slashes, etc). - */ - protected abstract String getMempoolApiHostname(); - - private static Optional args(Environment env) { - return Optional.ofNullable( - env.getProperty(CommandLinePropertySource.DEFAULT_NON_OPTION_ARGS_PROPERTY_NAME, String[].class)); - } - - private static int maxBlocks(Environment env) { - return args(env) - .filter(args -> args.length >= 1) - .map(args -> Integer.valueOf(args[0])) - .orElse(DEFAULT_MAX_BLOCKS); - } - - private static long refreshInterval(Environment env) { - return args(env) - .filter(args -> args.length >= 2) - .map(args -> Integer.valueOf(args[1])) - .orElse(DEFAULT_REFRESH_INTERVAL); - } - - @Primary - @Component - @Order(1) - public static class First extends MempoolFeeRateProvider { - - public First(Environment env) { - super(env); - } - - protected String getMempoolApiHostname() { - // This is the primary instance, so if no API point is set in - // application.properties file, then it defaults to mempool.space - // This ensures there is at least one provider attempting to connect, - // even if the properties file is corrupt or empty - return env.getProperty(MEMPOOL_HOSTNAME_KEY_1, "mempool.space"); - } - } - - @Component - @Order(2) - @ConditionalOnProperty(name = MEMPOOL_HOSTNAME_KEY_2) - public static class Second extends MempoolFeeRateProvider { - - public Second(Environment env) { - super(env); - } - - protected String getMempoolApiHostname() { - return env.getProperty(MEMPOOL_HOSTNAME_KEY_2); - } - } - - @Component - @Order(3) - @ConditionalOnProperty(name = MEMPOOL_HOSTNAME_KEY_3) - public static class Third extends MempoolFeeRateProvider { - - public Third(Environment env) { - super(env); - } - - protected String getMempoolApiHostname() { - return env.getProperty(MEMPOOL_HOSTNAME_KEY_3); - } - } - - @Component - @Order(4) - @ConditionalOnProperty(name = MEMPOOL_HOSTNAME_KEY_4) - public static class Fourth extends MempoolFeeRateProvider { - - public Fourth(Environment env) { - super(env); - } - - protected String getMempoolApiHostname() { - return env.getProperty(MEMPOOL_HOSTNAME_KEY_4); - } - } - - @Component - @Order(5) - @ConditionalOnProperty(name = MEMPOOL_HOSTNAME_KEY_5) - public static class Fifth extends MempoolFeeRateProvider { - - public Fifth(Environment env) { - super(env); - } - - protected String getMempoolApiHostname() { - return env.getProperty(MEMPOOL_HOSTNAME_KEY_5); - } - } - - @RestController - class Controller extends PriceController { - - @GetMapping(path = "/getParams") - public String getParams() { - return String.format("%s;%s", maxBlocks, refreshInterval.toMillis()); - } - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRate.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRate.java deleted file mode 100644 index 78bc5ff21bb..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/ExchangeRate.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.math.BigDecimal; - -import java.util.Date; -import java.util.Objects; - -/** - * A value object representing the spot price in bitcoin for a given currency at a given - * time as reported by a given provider. - */ -public class ExchangeRate { - - private final String currency; - private final double price; - private final long timestamp; - private final String provider; - - public ExchangeRate(String currency, BigDecimal price, Date timestamp, String provider) { - this( - currency, - price.doubleValue(), - timestamp.getTime(), - provider - ); - } - - public ExchangeRate(String currency, double price, long timestamp, String provider) { - this.currency = currency; - this.price = price; - this.timestamp = timestamp; - this.provider = provider; - } - - @JsonProperty(value = "currencyCode", index = 1) - public String getCurrency() { - return currency; - } - - @JsonProperty(value = "price", index = 2) - public double getPrice() { - return this.price; - } - - @JsonProperty(value = "timestampSec", index = 3) - public long getTimestamp() { - return this.timestamp; - } - - @JsonProperty(value = "provider", index = 4) - public String getProvider() { - return provider; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ExchangeRate exchangeRate = (ExchangeRate) o; - return Double.compare(exchangeRate.price, price) == 0 && - timestamp == exchangeRate.timestamp && - Objects.equals(currency, exchangeRate.currency) && - Objects.equals(provider, exchangeRate.provider); - } - - @Override - public int hashCode() { - return Objects.hash(currency, price, timestamp, provider); - } - - @Override - public String toString() { - return "ExchangeRate{" + - "currency='" + currency + '\'' + - ", price=" + price + - ", timestamp=" + timestamp + - ", provider=" + provider + - '}'; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRateController.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRateController.java deleted file mode 100644 index f4c01765a88..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/ExchangeRateController.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot; - -import bisq.price.PriceController; -import bisq.price.mining.FeeRateService; - -import bisq.common.config.Config; - -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Map; - -@RestController -class ExchangeRateController extends PriceController { - - private final ExchangeRateService exchangeRateService; - private final FeeRateService feeRateService; - - public ExchangeRateController(ExchangeRateService exchangeRateService, FeeRateService feeRateService) { - this.exchangeRateService = exchangeRateService; - this.feeRateService = feeRateService; - } - - @GetMapping(path = "/getAllMarketPrices") - public Map getAllMarketPrices() { - Map retVal = exchangeRateService.getAllMarketPrices(); - - // add the fee info to results - feeRateService.getFees().forEach((key, value) -> { - retVal.put(translateFieldName(key), value); - }); - - return retVal; - } - - static String translateFieldName(String name) { - if (name.equals(Config.LEGACY_FEE_DATAMAP)) - name = Config.BTC_FEE_INFO; // name changed for clarity - return name; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java deleted file mode 100644 index dae93a2c0ff..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot; - -import bisq.price.PriceProvider; - -import bisq.core.locale.CurrencyUtil; -import bisq.core.locale.TradeCurrency; - -import org.knowm.xchange.Exchange; -import org.knowm.xchange.ExchangeFactory; -import org.knowm.xchange.currency.Currency; -import org.knowm.xchange.currency.CurrencyPair; -import org.knowm.xchange.dto.marketdata.Ticker; -import org.knowm.xchange.exceptions.ExchangeException; -import org.knowm.xchange.exceptions.NotYetImplementedForExchangeException; -import org.knowm.xchange.service.marketdata.MarketDataService; -import org.knowm.xchange.service.marketdata.params.CurrencyPairsParam; -import org.knowm.xchange.service.marketdata.params.Params; - -import org.springframework.core.env.Environment; - -import java.time.Duration; - -import java.io.IOException; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Abstract base class for providers of bitcoin {@link ExchangeRate} data. Implementations - * are marked with the {@link org.springframework.stereotype.Component} annotation in - * order to be discovered via classpath scanning. If multiple - * {@link ExchangeRateProvider}s retrieve rates for the same currency, then the - * {@link ExchangeRateService} will average them out and expose an aggregate rate. - * - * @see ExchangeRateService#getAllMarketPrices() - */ -public abstract class ExchangeRateProvider extends PriceProvider> { - - private static Set SUPPORTED_CRYPTO_CURRENCIES = new HashSet<>(); - private static Set SUPPORTED_FIAT_CURRENCIES = new HashSet<>(); - private Set providerExclusionList = new HashSet<>(); - private final String name; - private final String prefix; - private final Environment env; - - public ExchangeRateProvider(Environment env, String name, String prefix, Duration refreshInterval) { - super(refreshInterval); - this.name = name; - this.prefix = prefix; - this.env = env; - List excludedByProvider = - Arrays.asList(env.getProperty("bisq.price.fiatcurrency.excludedByProvider", "") - .toUpperCase().trim().split("\\s*,\\s*")); - for (String s: excludedByProvider) { - String[] splits = s.split(":"); - if (splits.length == 2 && splits[0].equalsIgnoreCase(name) && CurrencyUtil.isFiatCurrency(splits[1])) { - providerExclusionList.add(splits[1]); - } - } - if (providerExclusionList.size() > 0) { - log.info("{} specific exclusion list={}", name, providerExclusionList.toString()); - } - } - - public Set getSupportedFiatCurrencies() { - if (SUPPORTED_FIAT_CURRENCIES.isEmpty()) { // one-time initialization - List excludedFiatCurrencies = - Arrays.asList(env.getProperty("bisq.price.fiatcurrency.excluded", "") - .toUpperCase().trim().split("\\s*,\\s*")); - String validatedExclusionList = excludedFiatCurrencies.stream() - .filter(ccy -> !ccy.isEmpty()) - .filter(CurrencyUtil::isFiatCurrency) - .collect(Collectors.toList()).toString(); - SUPPORTED_FIAT_CURRENCIES = CurrencyUtil.getAllSortedFiatCurrencies().stream() - .map(TradeCurrency::getCode) - .filter(ccy -> !validatedExclusionList.contains(ccy.toUpperCase())) - .collect(Collectors.toSet()); - log.info("fiat currencies excluded: {}", validatedExclusionList); - log.info("fiat currencies supported: {}", SUPPORTED_FIAT_CURRENCIES.size()); - } - // filter out any provider specific ccy exclusions - return SUPPORTED_FIAT_CURRENCIES.stream() - .filter(ccy -> !providerExclusionList.contains(ccy.toUpperCase())) - .collect(Collectors.toSet()); - } - - public Set getSupportedCryptoCurrencies() { - if (SUPPORTED_CRYPTO_CURRENCIES.isEmpty()) { // one-time initialization - List excludedCryptoCurrencies = - Arrays.asList(env.getProperty("bisq.price.cryptocurrency.excluded", "") - .toUpperCase().trim().split("\\s*,\\s*")); - String validatedExclusionList = excludedCryptoCurrencies.stream() - .filter(ccy -> !ccy.isEmpty()) - .filter(CurrencyUtil::isCryptoCurrency) - .collect(Collectors.toList()).toString(); - SUPPORTED_CRYPTO_CURRENCIES = CurrencyUtil.getAllSortedCryptoCurrencies().stream() - .map(TradeCurrency::getCode) - .filter(ccy -> !validatedExclusionList.contains(ccy.toUpperCase())) - .collect(Collectors.toSet()); - log.info("crypto currencies excluded: {}", validatedExclusionList); - log.info("crypto currencies supported: {}", SUPPORTED_CRYPTO_CURRENCIES.size()); - } - return SUPPORTED_CRYPTO_CURRENCIES; - } - - public String getName() { - return name; - } - - public String getPrefix() { - return prefix; - } - - @Override - protected void onRefresh() { - get().stream() - .filter(e -> "USD".equals(e.getCurrency()) || "LTC".equals(e.getCurrency())) - .forEach(e -> log.info("BTC/{}: {}", e.getCurrency(), e.getPrice())); - } - - /** - * @param exchangeClass Class of the {@link Exchange} for which the rates should be - * polled - * @return Exchange rates for Bisq-supported fiat currencies and altcoins in the - * specified {@link Exchange} - * - * @see CurrencyUtil#getAllSortedFiatCurrencies() - * @see CurrencyUtil#getAllSortedCryptoCurrencies() - */ - protected Set doGet(Class exchangeClass) { - Set result = new HashSet(); - - // Initialize XChange objects - Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exchangeClass.getName()); - MarketDataService marketDataService = exchange.getMarketDataService(); - - // Retrieve all currency pairs supported by the exchange - List allCurrencyPairsOnExchange = exchange.getExchangeSymbols(); - - // Find out which currency pairs we are interested in polling ("desired pairs") - // This will be the intersection of: - // 1) the pairs available on the exchange, and - // 2) the pairs Bisq considers relevant / valid - // This will result in two lists of desired pairs (fiat and alts) - - // Find the desired fiat pairs (pair format is BTC-FIAT) - List desiredFiatPairs = allCurrencyPairsOnExchange.stream() - .filter(cp -> cp.base.equals(Currency.BTC)) - .filter(cp -> getSupportedFiatCurrencies().contains(cp.counter.getCurrencyCode())) - .collect(Collectors.toList()); - - // Find the desired altcoin pairs (pair format is ALT-BTC) - List desiredCryptoPairs = allCurrencyPairsOnExchange.stream() - .filter(cp -> cp.counter.equals(Currency.BTC)) - .filter(cp -> getSupportedCryptoCurrencies().contains(cp.base.getCurrencyCode())) - .collect(Collectors.toList()); - - // Retrieve in bulk all tickers offered by the exchange - // The benefits of this approach (vs polling each ticker) are twofold: - // 1) the polling of the exchange is faster (one HTTP call vs several) - // 2) it's easier to stay below any API rate limits the exchange might have - List tickersRetrievedFromExchange = new ArrayList<>(); - try { - tickersRetrievedFromExchange = marketDataService.getTickers(new CurrencyPairsParam() { - - /** - * The {@link MarketDataService#getTickers(Params)} interface requires a - * {@link CurrencyPairsParam} argument when polling for tickers in bulk. - * This parameter is meant to indicate a list of currency pairs for which - * the tickers should be polled. However, the actual implementations for - * the different exchanges differ, for example: - * - some will ignore it (and retrieve all available tickers) - * - some will require it (and will fail if a null or empty list is given) - * - some will properly handle it - * - * We take a simplistic approach, namely: - * - for providers that require such a filter, specify one - * - for all others, do not specify one - * - * We make this distinction using - * {@link ExchangeRateProvider#requiresFilterDuringBulkTickerRetrieval} - * - * @return Filter (list of desired currency pairs) to be used during bulk - * ticker retrieval - */ - @Override - public Collection getCurrencyPairs() { - // If required by the exchange implementation, specify a filter - // (list of pairs which should be retrieved) - if (requiresFilterDuringBulkTickerRetrieval()) { - return Stream.of(desiredFiatPairs, desiredCryptoPairs) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - } - - // Otherwise, specify an empty list, indicating that the API should - // simply return all available tickers - return Collections.emptyList(); - } - }); - - if (tickersRetrievedFromExchange.isEmpty()) { - // If the bulk ticker retrieval went through, but no tickers were - // retrieved, this is a strong indication that this specific exchange - // needs a specific list of pairs given as argument, for bulk retrieval to - // work. See requiresFilterDuringBulkTickerRetrieval() - throw new IllegalArgumentException("No tickers retrieved, " + - "exchange requires explicit filter argument during bulk retrieval?"); - } - } catch (NotYetImplementedForExchangeException e) { - // Thrown when a provider has no marketDataService.getTickers() implementation - // either because the exchange API does not provide it, or because it has not - // been implemented yet in the knowm xchange library - - // In this case (retrieval of bulk tickers is not possible) retrieve the - // tickers one by one - List finalTickersRetrievedFromExchange = tickersRetrievedFromExchange; - Stream.of(desiredFiatPairs, desiredCryptoPairs) - .flatMap(Collection::stream) - .collect(Collectors.toList()) - .forEach(cp -> { - try { - - // This is done in a loop, and can therefore result in a burst - // of API calls. Some exchanges do not allow bursts - // A simplistic solution is to delay every call by 1 second - // TODO Switch to using a more elegant solution (per exchange) - // like ResilienceSpecification (needs knowm xchange libs v5) - if (getMarketDataCallDelay() > 0) { - Thread.sleep(getMarketDataCallDelay()); - } - - Ticker ticker = marketDataService.getTicker(cp); - finalTickersRetrievedFromExchange.add(ticker); - - } catch (IOException | InterruptedException ioException) { - ioException.printStackTrace(); - log.error("Could not query tickers for " + getName(), e); - } - }); - } catch (ExchangeException | // Errors reported by the exchange (rate limit, etc) - IOException | // Errors while trying to connect to the API (timeouts, etc) - // Potential error when integrating new exchange (hints that exchange - // provider implementation needs to overwrite - // requiresFilterDuringBulkTickerRetrieval() and have it return true ) - IllegalArgumentException e) { - // Catch and handle all other possible exceptions - // If there was a problem with polling this exchange, return right away, - // since there are no results to parse and process - log.error("Could not query tickers for provider " + getName(), e); - return result; - } - - // Create an ExchangeRate for each desired currency pair ticker that was retrieved - Predicate isDesiredFiatPair = t -> desiredFiatPairs.contains(t.getCurrencyPair()); - Predicate isDesiredCryptoPair = t -> desiredCryptoPairs.contains(t.getCurrencyPair()); - tickersRetrievedFromExchange.stream() - .filter(isDesiredFiatPair.or(isDesiredCryptoPair)) // Only consider desired pairs - .forEach(t -> { - // All tickers here match all requirements - - // We have two kinds of currency pairs, BTC-FIAT and ALT-BTC - // In the first one, BTC is the first currency of the pair - // In the second type, BTC is listed as the second currency - // Distinguish between the two and create ExchangeRates accordingly - - // In every Bisq ExchangeRate, BTC is one currency in the pair - // Extract the other currency from the ticker, to create ExchangeRates - String otherExchangeRateCurrency; - if (t.getCurrencyPair().base.equals(Currency.BTC)) { - otherExchangeRateCurrency = t.getCurrencyPair().counter.getCurrencyCode(); - } else { - otherExchangeRateCurrency = t.getCurrencyPair().base.getCurrencyCode(); - } - - result.add(new ExchangeRate( - otherExchangeRateCurrency, - t.getLast(), - // Some exchanges do not provide timestamps - t.getTimestamp() == null ? new Date() : t.getTimestamp(), - this.getName() - )); - }); - - return result; - } - - /** - * Specifies optional delay between certain kind of API calls that can result in - * bursts. We want to avoid bursts, because this can cause certain exchanges to - * temporarily restrict access to the pricenode IP. - * - * @return Amount of milliseconds of delay between marketDataService.getTicker calls. - * By default 0, but can be overwritten by each provider. - */ - protected long getMarketDataCallDelay() { - return 0; - } - - /** - * @return Whether or not the bulk retrieval of tickers from the exchange requires an - * explicit filter (list of desired pairs) or not. If true, the - * {@link MarketDataService#getTickers(Params)} call will be constructed and given as - * argument, which acts as a filter indicating for which pairs the ticker should be - * retrieved. If false, {@link MarketDataService#getTickers(Params)} will be called - * with an empty argument, indicating that the API should simply return all available - * tickers on the exchange - */ - protected boolean requiresFilterDuringBulkTickerRetrieval() { - return false; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRateService.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRateService.java deleted file mode 100644 index e1300b6d5a2..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/ExchangeRateService.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot; - -import org.springframework.stereotype.Service; - -import java.math.BigDecimal; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.OptionalDouble; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Arrays.asList; - -/** - * High-level {@link ExchangeRate} data operations. - */ -@Service -class ExchangeRateService { - protected final Logger log = LoggerFactory.getLogger(this.getClass()); - - private final List providers; - - /** - * Construct an {@link ExchangeRateService} with a list of all - * {@link ExchangeRateProvider} implementations discovered via classpath scanning. - * - * @param providers all {@link ExchangeRateProvider} implementations in ascending - * order of precedence - */ - public ExchangeRateService(List providers) { - this.providers = providers; - } - - public Map getAllMarketPrices() { - Map metadata = new LinkedHashMap<>(); - Map aggregateExchangeRates = getAggregateExchangeRates(); - - providers.forEach(p -> { - if (p.get() == null) - return; - Set exchangeRates = p.get(); - - // Specific metadata fields for specific providers are expected by the client, - // mostly for historical reasons - // Therefore, add metadata fields for all known providers - // Rates are encapsulated in the "data" map below - metadata.putAll(getMetadata(p, exchangeRates)); - }); - - LinkedHashMap result = new LinkedHashMap<>(); - result.putAll(metadata); - // Use a sorted list by currency code to make comparision of json data between - // different price nodes easier - List values = new ArrayList<>(aggregateExchangeRates.values()); - values.sort(Comparator.comparing(ExchangeRate::getCurrency)); - result.put("data", values); - - return result; - } - - /** - * For each currency, create an aggregate {@link ExchangeRate} based on the currency's - * rates from all providers. If multiple providers have rates for the currency, then - * aggregate price = average of retrieved prices. If a single provider has rates for - * the currency, then aggregate price = the rate from that provider. - * - * @return Aggregate {@link ExchangeRate}s based on info from all providers, indexed - * by currency code - */ - private Map getAggregateExchangeRates() { - Map aggregateExchangeRates = new HashMap<>(); - - // Query all providers and collect all exchange rates, grouped by currency code - // key = currency code - // value = list of exchange rates - Map> currencyCodeToExchangeRates = getCurrencyCodeToExchangeRates(); - - // For each currency code, calculate aggregate rate - currencyCodeToExchangeRates.forEach((currencyCode, exchangeRateList) -> { - if (exchangeRateList.isEmpty()) { - // If the map was built incorrectly and this currency points to an empty - // list of rates, skip it - return; - } - - ExchangeRate aggregateExchangeRate; - if (exchangeRateList.size() == 1) { - // If a single provider has rates for this currency, then aggregate = rate - // from that provider - aggregateExchangeRate = exchangeRateList.get(0); - } else { - // If multiple providers have rates for this currency, then - // aggregate = average of the rates - OptionalDouble opt = exchangeRateList.stream().mapToDouble(ExchangeRate::getPrice).average(); - // List size > 1, so opt is always set - double priceAvg = opt.orElseThrow(IllegalStateException::new); - - aggregateExchangeRate = new ExchangeRate( - currencyCode, - BigDecimal.valueOf(priceAvg), - new Date(), // timestamp = time when avg is calculated - "Bisq-Aggregate"); - } - aggregateExchangeRates.put(aggregateExchangeRate.getCurrency(), aggregateExchangeRate); - }); - - return aggregateExchangeRates; - } - - /** - * @return All {@link ExchangeRate}s from all providers, grouped by currency code - */ - private Map> getCurrencyCodeToExchangeRates() { - Map> currencyCodeToExchangeRates = new HashMap<>(); - for (ExchangeRateProvider p : providers) { - if (p.get() == null) - continue; - for (ExchangeRate exchangeRate : p.get()) { - String currencyCode = exchangeRate.getCurrency(); - if (currencyCodeToExchangeRates.containsKey(currencyCode)) { - List l = new ArrayList<>(currencyCodeToExchangeRates.get(currencyCode)); - l.add(exchangeRate); - currencyCodeToExchangeRates.put(currencyCode, l); - } else { - currencyCodeToExchangeRates.put(currencyCode, asList(exchangeRate)); - } - } - } - return currencyCodeToExchangeRates; - } - - private Map getMetadata(ExchangeRateProvider provider, Set exchangeRates) { - Map metadata = new LinkedHashMap<>(); - - // In case a provider is not available we still want to deliver the data of the - // other providers, so we catch a possible exception and leave timestamp at 0. The - // Bisq app will check if the timestamp is in a tolerance window and if it is too - // old it will show that the price is not available. - long timestamp = 0; - try { - timestamp = getTimestamp(provider, exchangeRates); - } catch (Throwable t) { - log.error(t.toString()); - if (log.isDebugEnabled()) - t.printStackTrace(); - } - - String prefix = provider.getPrefix(); - metadata.put(prefix + "Ts", timestamp); - metadata.put(prefix + "Count", exchangeRates.size()); - - return metadata; - } - - private long getTimestamp(ExchangeRateProvider provider, Set exchangeRates) { - return exchangeRates.stream() - .filter(e -> provider.getName().equals(e.getProvider())) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No exchange rate data found for " + provider.getName())) - .getTimestamp(); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/BTCMarkets.java b/pricenode/src/main/java/bisq/price/spot/providers/BTCMarkets.java deleted file mode 100644 index e9b8e843358..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/BTCMarkets.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.btcmarkets.BTCMarketsExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class BTCMarkets extends ExchangeRateProvider { - - public BTCMarkets(Environment env) { - super(env, "BTCMARKETS", "btcmarkets", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: AUD - // Supported alts: ETH, LTC - return doGet(BTCMarketsExchange.class); - } - -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Binance.java b/pricenode/src/main/java/bisq/price/spot/providers/Binance.java deleted file mode 100644 index d630b808133..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Binance.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.binance.BinanceExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Binance extends ExchangeRateProvider { - - public Binance(Environment env) { - super(env, "BINANCE", "binance", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: EUR, GBP, NGN, RUB, TRY, UAH, ZAR - // Supported alts: BEAM, DAI, DASH, DCR, DOGE, ETC, ETH, LTC, NAV, PIVX, XMR, XZC, - // ZEC, ZEN - return doGet(BinanceExchange.class); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/BitcoinAverage.java b/pricenode/src/main/java/bisq/price/spot/providers/BitcoinAverage.java deleted file mode 100644 index db33f9da702..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/BitcoinAverage.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.HashSet; -import java.util.Set; - -/** - * Stub implementation (similar to #CoinMarketCap) for backward compatibility with legacy - * Bisq clients - */ -@Component -class BitcoinAverage extends ExchangeRateProvider { - - public BitcoinAverage(Environment env) { - // Simulate a deactivated BitcoinAverage provider - // We still need the class to exist and be registered as a provider though, - // because the returned data structure must contain the "btcAverageTs" key - // for backward compatibility with Bisq clients which hardcode that key - super(env, "BA", "btcAverage", Duration.ofMinutes(100)); - } - - /** - * @see CoinMarketCap#doGet() - * @return - */ - @Override - public Set doGet() { - - HashSet exchangeRates = new HashSet<>(); - exchangeRates.add(new ExchangeRate("NON_EXISTING_SYMBOL_BA", 0, 0L, getName())); - return exchangeRates; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Bitfinex.java b/pricenode/src/main/java/bisq/price/spot/providers/Bitfinex.java deleted file mode 100644 index cc8f62f3950..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Bitfinex.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.bitfinex.BitfinexExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Bitfinex extends ExchangeRateProvider { - - public Bitfinex(Environment env) { - super(env, "BITFINEX", "bitfinex", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: EUR, GBP, JPY, USD - // Supported alts: DAI, ETC, ETH, LTC, XMR, ZEC - return doGet(BitfinexExchange.class); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Bitflyer.java b/pricenode/src/main/java/bisq/price/spot/providers/Bitflyer.java deleted file mode 100644 index c2532847fb0..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Bitflyer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.bitflyer.BitflyerExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Bitflyer extends ExchangeRateProvider { - - public Bitflyer(Environment env) { - super(env, "BITFLYER", "bitflyer", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: JPY - // Supported alts: ETH - return doGet(BitflyerExchange.class); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Bitstamp.java b/pricenode/src/main/java/bisq/price/spot/providers/Bitstamp.java deleted file mode 100644 index 798f6aa95c8..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Bitstamp.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.bitstamp.BitstampExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Bitstamp extends ExchangeRateProvider { - - public Bitstamp(Environment env) { - super(env, "BITSTAMP", "bitstamp", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: EUR, GBP, USD - // Supported alts: ETH, LTC - return doGet(BitstampExchange.class); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/CoinGecko.java b/pricenode/src/main/java/bisq/price/spot/providers/CoinGecko.java deleted file mode 100644 index 556001fc378..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/CoinGecko.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; -import bisq.price.util.coingecko.CoinGeckoMarketData; - -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.core.env.Environment; -import org.springframework.http.RequestEntity; -import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -import java.time.Duration; - -import java.math.BigDecimal; -import java.math.RoundingMode; - -import java.util.Date; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -@Component -class CoinGecko extends ExchangeRateProvider { - - private final RestTemplate restTemplate = new RestTemplate(); - - public CoinGecko(Environment env) { - super(env, "COINGECKO", "coingecko", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - - // Rate limit for the CoinGecko API is 10 calls each second per IP address - // We retrieve all rates in bulk, so we only make 1 call per provider poll - - Set result = new HashSet(); - - Predicate isDesiredFiatPair = t -> getSupportedFiatCurrencies().contains(t.getKey()); - Predicate isDesiredCryptoPair = t -> getSupportedCryptoCurrencies().contains(t.getKey()); - - getMarketData().getRates().entrySet().stream() - .filter(isDesiredFiatPair.or(isDesiredCryptoPair)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - .forEach((key, ticker) -> { - - boolean useInverseRate = false; - if (getSupportedCryptoCurrencies().contains(key)) { - // Use inverse rate for alts, because the API returns the - // conversion rate in the opposite direction than what we need - // API returns the BTC/Alt rate, we need the Alt/BTC rate - useInverseRate = true; - } - - BigDecimal rate = ticker.getValue(); - // Find the inverse rate, while using enough decimals to reflect very - // small exchange rates - BigDecimal inverseRate = (rate.compareTo(BigDecimal.ZERO) > 0) ? - BigDecimal.ONE.divide(rate, 8, RoundingMode.HALF_UP) : - BigDecimal.ZERO; - - result.add(new ExchangeRate( - key, - (useInverseRate ? inverseRate : rate), - new Date(), - this.getName() - )); - }); - - return result; - } - - private CoinGeckoMarketData getMarketData() { - return restTemplate.exchange( - RequestEntity - .get(UriComponentsBuilder - .fromUriString("https://api.coingecko.com/api/v3/exchange_rates").build() - .toUri()) - .build(), - new ParameterizedTypeReference() { - } - ).getBody(); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/CoinMarketCap.java b/pricenode/src/main/java/bisq/price/spot/providers/CoinMarketCap.java deleted file mode 100644 index d18ee649c7a..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/CoinMarketCap.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.HashSet; -import java.util.Set; - -/** - * Stub implementation of CoinMarketCap price provider to prevent NullPointerExceptions within legacy clients - */ -@Component -class CoinMarketCap extends ExchangeRateProvider { - - public CoinMarketCap(Environment env) { - super(env, "CMC", "coinmarketcap", Duration.ofMinutes(5)); // large data structure, so don't request it too often - } - - /** - * Returns a Set with a non existing symbol for the CoinMarketCap price provider. - * Price data of CMC provider is not used in the client anymore, except for the last update timestamp. - * To prevent a unnecessary warning log in that case we have to pass at least one element. - * - * @return Empty Set - */ - @Override - public Set doGet() { - HashSet exchangeRates = new HashSet<>(); - exchangeRates.add(new ExchangeRate("NON_EXISTING_SYMBOL", 0, 0L, "CMC")); - return exchangeRates; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/CoinbasePro.java b/pricenode/src/main/java/bisq/price/spot/providers/CoinbasePro.java deleted file mode 100644 index 663fee8e39f..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/CoinbasePro.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.coinbasepro.CoinbaseProExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class CoinbasePro extends ExchangeRateProvider { - - public CoinbasePro(Environment env) { - super(env, "COINBASEPRO", "coinbasepro", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: EUR, USD, GBP - // Supported alts: DASH, DOGE, ETC, ETH, LTC, ZEC, ZEN - return doGet(CoinbaseProExchange.class); - } - - @Override - protected boolean requiresFilterDuringBulkTickerRetrieval() { - return true; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Coinone.java b/pricenode/src/main/java/bisq/price/spot/providers/Coinone.java deleted file mode 100644 index bf125756732..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Coinone.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.coinone.CoinoneExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Coinone extends ExchangeRateProvider { - - public Coinone(Environment env) { - super(env, "COINONE", "coinone", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: KRW - // Supported alts: - - return doGet(CoinoneExchange.class); - } - -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/IndependentReserve.java b/pricenode/src/main/java/bisq/price/spot/providers/IndependentReserve.java deleted file mode 100644 index db1b881c228..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/IndependentReserve.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.independentreserve.IndependentReserveExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class IndependentReserve extends ExchangeRateProvider { - - public IndependentReserve(Environment env) { - super(env, "IndependentReserve", "independentreserve", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: AUD, NZD (New Zealand Dollar), USD - // Supported alts: - - return doGet(IndependentReserveExchange.class); - } - -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Kraken.java b/pricenode/src/main/java/bisq/price/spot/providers/Kraken.java deleted file mode 100644 index 0d1d2adc8ce..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Kraken.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.kraken.KrakenExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Kraken extends ExchangeRateProvider { - - public Kraken(Environment env) { - super(env, "KRAKEN", "kraken", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: AUD, CAD, CHF, EUR, GBP, JPY, USD - // Supported alts: DASH, DOGE, ETC, ETH, LTC, XMR, ZEC - return doGet(KrakenExchange.class); - } - - @Override - protected boolean requiresFilterDuringBulkTickerRetrieval() { - return true; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Luno.java b/pricenode/src/main/java/bisq/price/spot/providers/Luno.java deleted file mode 100644 index 451f6ba864d..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Luno.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.luno.LunoExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Luno extends ExchangeRateProvider { - - public Luno(Environment env) { - super(env, "LUNO", "luno", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: IDR (Indonesian rupiah), MYR (Malaysian ringgit), - // NGN (Nigerian Naira), ZAR (South African rand) - // Supported alts: - - return doGet(LunoExchange.class); - } - - @Override - protected long getMarketDataCallDelay() { - // Luno allows only 1 MarketData call per second - // (see https://www.luno.com/en/developers/api ) - return 1000; - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/MercadoBitcoin.java b/pricenode/src/main/java/bisq/price/spot/providers/MercadoBitcoin.java deleted file mode 100644 index 7ecbbb20a64..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/MercadoBitcoin.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.mercadobitcoin.MercadoBitcoinExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class MercadoBitcoin extends ExchangeRateProvider { - - public MercadoBitcoin(Environment env) { - super(env, "MercadoBitcoin", "mercadobitcoin", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: BRL (Brazilian Real) - // Supported alts: - - return doGet(MercadoBitcoinExchange.class); - } - -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Paribu.java b/pricenode/src/main/java/bisq/price/spot/providers/Paribu.java deleted file mode 100644 index 8498aff68ee..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Paribu.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.paribu.ParibuExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Paribu extends ExchangeRateProvider { - - public Paribu(Environment env) { - super(env, "PARIBU", "paribu", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: TRY (Turkish Lira) - // Supported alts: - - return doGet(ParibuExchange.class); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Poloniex.java b/pricenode/src/main/java/bisq/price/spot/providers/Poloniex.java deleted file mode 100644 index 6664e7ea3a5..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Poloniex.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.poloniex.PoloniexExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Poloniex extends ExchangeRateProvider { - - public Poloniex(Environment env) { - super(env, "POLO", "poloniex", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: - - // Supported alts: DASH, DCR, DOGE, ETC, ETH, LTC, XMR, ZEC - return doGet(PoloniexExchange.class); - } -} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Quoine.java b/pricenode/src/main/java/bisq/price/spot/providers/Quoine.java deleted file mode 100644 index 745181727f8..00000000000 --- a/pricenode/src/main/java/bisq/price/spot/providers/Quoine.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import org.knowm.xchange.quoine.QuoineExchange; - -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.time.Duration; - -import java.util.Set; - -@Component -class Quoine extends ExchangeRateProvider { - - public Quoine(Environment env) { - super(env, "QUOINE", "quoine", Duration.ofMinutes(1)); - } - - @Override - public Set doGet() { - // Supported fiat: AUD, CNY, EUR, HKD, IDR, INR, JPY, PHP, SGD, USD - // Supported alts: ETH - return doGet(QuoineExchange.class); - } - - @Override - protected boolean requiresFilterDuringBulkTickerRetrieval() { - return true; - } -} diff --git a/pricenode/src/main/java/bisq/price/util/Altcoins.java b/pricenode/src/main/java/bisq/price/util/Altcoins.java deleted file mode 100644 index e7917ce8c64..00000000000 --- a/pricenode/src/main/java/bisq/price/util/Altcoins.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.util; - -import bisq.core.locale.CurrencyUtil; -import bisq.core.locale.TradeCurrency; - -import java.util.Set; -import java.util.stream.Collectors; - -public abstract class Altcoins { - - public static final Set ALL_SUPPORTED = - CurrencyUtil.getAllSortedCryptoCurrencies().stream() - .map(TradeCurrency::getCode) - .collect(Collectors.toSet()); -} diff --git a/pricenode/src/main/java/bisq/price/util/bitpay/BitpayMarketData.java b/pricenode/src/main/java/bisq/price/util/bitpay/BitpayMarketData.java deleted file mode 100644 index 73ab946a9bc..00000000000 --- a/pricenode/src/main/java/bisq/price/util/bitpay/BitpayMarketData.java +++ /dev/null @@ -1,13 +0,0 @@ -package bisq.price.util.bitpay; - - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class BitpayMarketData { - - private BitpayTicker[] data; - -} diff --git a/pricenode/src/main/java/bisq/price/util/bitpay/BitpayTicker.java b/pricenode/src/main/java/bisq/price/util/bitpay/BitpayTicker.java deleted file mode 100644 index 8cc63773ac3..00000000000 --- a/pricenode/src/main/java/bisq/price/util/bitpay/BitpayTicker.java +++ /dev/null @@ -1,18 +0,0 @@ -package bisq.price.util.bitpay; - -import java.math.BigDecimal; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class BitpayTicker { - - private String code; - - private String name; - - private BigDecimal rate; - -} diff --git a/pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoMarketData.java b/pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoMarketData.java deleted file mode 100644 index b96ba10dcaa..00000000000 --- a/pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoMarketData.java +++ /dev/null @@ -1,22 +0,0 @@ -package bisq.price.util.coingecko; - - -import java.util.Map; -import java.util.stream.Collectors; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class CoinGeckoMarketData { - - private Map rates; - - public void setRates(Map rates) { - // Convert keys to uppercase ("usd" -> "USD") when deserializing API response - this.rates = rates.entrySet().stream() - .collect(Collectors.toMap(entry -> entry.getKey().toUpperCase(), entry -> entry.getValue())); - } - -} diff --git a/pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoTicker.java b/pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoTicker.java deleted file mode 100644 index 2326203a100..00000000000 --- a/pricenode/src/main/java/bisq/price/util/coingecko/CoinGeckoTicker.java +++ /dev/null @@ -1,20 +0,0 @@ -package bisq.price.util.coingecko; - -import java.math.BigDecimal; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class CoinGeckoTicker { - - private String name; - - private String unit; - - private BigDecimal value; - - private String type; - -} diff --git a/pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaMarketData.java b/pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaMarketData.java deleted file mode 100644 index b5e366ddbcb..00000000000 --- a/pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaMarketData.java +++ /dev/null @@ -1,17 +0,0 @@ -package bisq.price.util.coinpaprika; - - -import java.util.Map; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class CoinpaprikaMarketData { - - // All other json fields can be ignored, we don't need them - - private Map quotes; - -} diff --git a/pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaTicker.java b/pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaTicker.java deleted file mode 100644 index 64bb3183120..00000000000 --- a/pricenode/src/main/java/bisq/price/util/coinpaprika/CoinpaprikaTicker.java +++ /dev/null @@ -1,16 +0,0 @@ -package bisq.price.util.coinpaprika; - -import java.math.BigDecimal; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class CoinpaprikaTicker { - - // All other json fields can be ignored, we don't need them - - private BigDecimal price; - -} diff --git a/pricenode/src/main/resources/application.properties b/pricenode/src/main/resources/application.properties deleted file mode 100644 index 276bad51d90..00000000000 --- a/pricenode/src/main/resources/application.properties +++ /dev/null @@ -1,12 +0,0 @@ -spring.jackson.serialization.indent_output=true - -# To enable another fee estimation endpoint, simply uncomment one of the following lines -# and set it to hostname exposing the fee estimation API -bisq.price.mining.providers.mempoolHostname.1=mempool.space -bisq.price.mining.providers.mempoolHostname.2=mempool.emzy.de -bisq.price.mining.providers.mempoolHostname.3=mempool.ninja -bisq.price.mining.providers.mempoolHostname.4=mempool.bisq.services -# bisq.price.mining.providers.mempoolHostname.5=someHostOrIP -bisq.price.fiatcurrency.excluded=LBP,ARS -bisq.price.fiatcurrency.excludedByProvider=HUOBI:BRL -bisq.price.cryptocurrency.excluded= diff --git a/pricenode/src/main/resources/banner.txt b/pricenode/src/main/resources/banner.txt deleted file mode 100644 index 08d29b7388b..00000000000 --- a/pricenode/src/main/resources/banner.txt +++ /dev/null @@ -1,6 +0,0 @@ - __ _ _ __ - / /_ (_)________ _ ____ _____(_)_______ ____ ____ ____/ /__ - / __ \/ / ___/ __ `/_____/ __ \/ ___/ / ___/ _ \/ __ \/ __ \/ __ / _ \ - / /_/ / (__ ) /_/ /_____/ /_/ / / / / /__/ __/ / / / /_/ / /_/ / __/ -/_.___/_/____/\__, / / .___/_/ /_/\___/\___/_/ /_/\____/\__,_/\___/ - /_/ /_/ ${application.formatted-version} diff --git a/pricenode/src/main/resources/logback.xml b/pricenode/src/main/resources/logback.xml deleted file mode 100644 index 02acbae6f9e..00000000000 --- a/pricenode/src/main/resources/logback.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - %highlight(%d{MMM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{15}: %msg %xEx%n) - - - - - - - - - - - diff --git a/pricenode/src/main/resources/version.txt b/pricenode/src/main/resources/version.txt deleted file mode 100644 index e87c17ec4a2..00000000000 --- a/pricenode/src/main/resources/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.7.2-SNAPSHOT diff --git a/pricenode/src/test/java/bisq/price/AbstractExchangeRateProviderTest.java b/pricenode/src/test/java/bisq/price/AbstractExchangeRateProviderTest.java deleted file mode 100644 index 911f45439d0..00000000000 --- a/pricenode/src/test/java/bisq/price/AbstractExchangeRateProviderTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package bisq.price; - -import bisq.price.spot.ExchangeRate; -import bisq.price.spot.ExchangeRateProvider; - -import com.google.common.collect.Sets; - -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -import static org.junit.Assert.assertTrue; - -@Slf4j -public abstract class AbstractExchangeRateProviderTest { - - protected void doGet_successfulCall(ExchangeRateProvider exchangeProvider) { - - // Use the XChange library to call the provider API, in order to retrieve the - // exchange rates. If the API call fails, or the response body cannot be parsed, - // the test will fail with an exception - Set retrievedExchangeRates = exchangeProvider.doGet(); - - // Log the valid exchange rates which were retrieved - // Useful when running the tests, to easily identify which exchanges provide - // useful pairs - retrievedExchangeRates.forEach(e -> log.info("Found exchange rate " + e.toString())); - - // Sanity checks - assertTrue(retrievedExchangeRates.size() > 0); - checkProviderCurrencyPairs(exchangeProvider, retrievedExchangeRates); - } - - /** - * Check that every retrieved currency pair is between BTC and either - * A) a fiat currency on the list of Bisq-supported fiat currencies, or - * B) an altcoin on the list of Bisq-supported altcoins - * - * @param retrievedExchangeRates Exchange rates retrieved from the provider - */ - private void checkProviderCurrencyPairs(ExchangeRateProvider exchangeProvider, Set retrievedExchangeRates) { - Set retrievedRatesCurrencies = retrievedExchangeRates.stream() - .map(ExchangeRate::getCurrency) - .collect(Collectors.toSet()); - - Set supportedFiatCurrenciesRetrieved = exchangeProvider.getSupportedFiatCurrencies().stream() - .filter(f -> retrievedRatesCurrencies.contains(f)) - .collect(Collectors.toCollection(TreeSet::new)); - log.info("Retrieved rates for supported fiat currencies: " + supportedFiatCurrenciesRetrieved); - - Set supportedCryptoCurrenciesRetrieved = exchangeProvider.getSupportedCryptoCurrencies().stream() - .filter(c -> retrievedRatesCurrencies.contains(c)) - .collect(Collectors.toCollection(TreeSet::new)); - log.info("Retrieved rates for supported altcoins: " + supportedCryptoCurrenciesRetrieved); - - Set supportedCurrencies = Sets.union( - exchangeProvider.getSupportedCryptoCurrencies(), - exchangeProvider.getSupportedFiatCurrencies()); - - Set unsupportedCurrencies = Sets.difference(retrievedRatesCurrencies, supportedCurrencies); - assertTrue("Retrieved exchange rates contain unsupported currencies: " + unsupportedCurrencies, - unsupportedCurrencies.isEmpty()); - } -} diff --git a/pricenode/src/test/java/bisq/price/mining/FeeRateServiceTest.java b/pricenode/src/test/java/bisq/price/mining/FeeRateServiceTest.java deleted file mode 100644 index 8f47584a413..00000000000 --- a/pricenode/src/test/java/bisq/price/mining/FeeRateServiceTest.java +++ /dev/null @@ -1,110 +0,0 @@ -package bisq.price.mining; - -import bisq.price.mining.providers.MempoolFeeRateProviderTest; - -import bisq.common.config.Config; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.junit.jupiter.api.Test; - -import static bisq.price.mining.providers.MempoolFeeRateProviderTest.buildDummyReachableMempoolFeeRateProvider; -import static bisq.price.mining.providers.MempoolFeeRateProviderTest.buildDummyUnreachableMempoolFeeRateProvider; -import static java.util.Arrays.asList; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; - -/** - * Tests the {@link bisq.price.mining.FeeRateService}, which can aggregate data from - * several {@link FeeRateProvider}s. - * @see MempoolFeeRateProviderTest - */ -public class FeeRateServiceTest { - - private static final Logger log = LoggerFactory.getLogger(FeeRateServiceTest.class); - - @Test - public void getFees_noWorkingProvider() { - // Several providers, but all unreachable - List listOfProviders = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - try { - listOfProviders.add(buildDummyUnreachableMempoolFeeRateProvider()); - } catch (Exception e) { - // Expected - log.info("We encountered an expected exception: " + e.getMessage()); - } - } - FeeRateService service = new FeeRateService(listOfProviders); - - Map retrievedData = service.getFees(); - - // Even with no working providers, we expect the service to return pre-configured - // minimum fee rate - doSanityChecksForRetrievedData(retrievedData, FeeRateProvider.MIN_FEE_RATE_FOR_TRADING); - } - - @Test - public void getFees_singleProvider_feeBelowMin() { - // One working provider, which returns a fee lower than the minimum - long providerFee = FeeRateProvider.MIN_FEE_RATE_FOR_TRADING - 3; - FeeRateService service = new FeeRateService( - Collections.singletonList( - buildDummyReachableMempoolFeeRateProvider(providerFee))); - - Map retrievedData = service.getFees(); - - // When the provider returns a value below the expected min, the service should - // return the min - doSanityChecksForRetrievedData(retrievedData, FeeRateProvider.MIN_FEE_RATE_FOR_TRADING); - } - - @Test - public void getFees_singleProvider_feeAboveMax() { - // One working provider, which returns a fee greater than the maximum - long providerFee = FeeRateProvider.MAX_FEE_RATE + 13; - FeeRateService service = new FeeRateService( - Collections.singletonList( - buildDummyReachableMempoolFeeRateProvider(providerFee))); - - Map retrievedData = service.getFees(); - - // When the provider returns a value above the expected max, the service should - // return the max - doSanityChecksForRetrievedData(retrievedData, FeeRateProvider.MAX_FEE_RATE); - } - - @Test - public void getFees_multipleProviders() { - // 3 providers, returning 1xMIN, 2xMIN, 3xMIN - FeeRateService service = new FeeRateService(asList( - buildDummyReachableMempoolFeeRateProvider(FeeRateProvider.MIN_FEE_RATE_FOR_TRADING * 1), - buildDummyReachableMempoolFeeRateProvider(FeeRateProvider.MIN_FEE_RATE_FOR_TRADING * 2), - buildDummyReachableMempoolFeeRateProvider(FeeRateProvider.MIN_FEE_RATE_FOR_TRADING * 3))); - - Map retrievedData = service.getFees(); - - // The service should then return the average, which is 2xMIN - doSanityChecksForRetrievedData(retrievedData, FeeRateProvider.MIN_FEE_RATE_FOR_TRADING * 2); - } - - /** - * Performs a few basic sanity checks on the returned data object - */ - private void doSanityChecksForRetrievedData(Map retrievedData, long expectedFeeRate) { - // Check if the response has the expected format. Since the timestamp is that of - // the average (not that of the individual fee rates reported by the individual - // providers), we always expect a non-zero timestamp - assertNotEquals(0L, retrievedData.get(Config.BTC_FEES_TS)); - - Map retrievedDataMap = (Map) retrievedData.get(Config.LEGACY_FEE_DATAMAP); - assertEquals(2, retrievedDataMap.size()); - assertEquals(expectedFeeRate, retrievedDataMap.get(Config.BTC_TX_FEE)); - } -} diff --git a/pricenode/src/test/java/bisq/price/mining/providers/MempoolFeeRateProviderTest.java b/pricenode/src/test/java/bisq/price/mining/providers/MempoolFeeRateProviderTest.java deleted file mode 100644 index e12140b43e6..00000000000 --- a/pricenode/src/test/java/bisq/price/mining/providers/MempoolFeeRateProviderTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.mining.providers; - -import bisq.price.mining.FeeRate; -import bisq.price.mining.FeeRateProvider; - -import org.springframework.core.env.Environment; -import org.springframework.core.env.StandardEnvironment; -import org.springframework.web.client.RestClientException; - -import java.time.Instant; - -import org.junit.jupiter.api.Test; - -import static java.lang.Thread.sleep; -import static org.junit.Assert.assertTrue; - -/** - * Tests specific to a {@link MempoolFeeRateProvider} which queries one API endpoint. For - * tests related to managing parallel fee API endpoints, see - * {@link bisq.price.mining.FeeRateServiceTest} - */ -public class MempoolFeeRateProviderTest { - - private static final Environment env = new StandardEnvironment(); - - @Test - public void doGet_successfulCall() { - MempoolFeeRateProvider feeRateProvider = new MempoolFeeRateProvider.First(env); - - // Make a call to the API, retrieve the recommended fee rate - // If the API call fails, or the response body cannot be parsed, the test will - // fail with an exception - FeeRate retrievedFeeRate = feeRateProvider.doGet(); - - // Check that the FeeRateProvider returns a fee within the defined parameters - assertTrue(retrievedFeeRate.getPrice() >= FeeRateProvider.MIN_FEE_RATE_FOR_TRADING); - assertTrue(retrievedFeeRate.getPrice() <= FeeRateProvider.MAX_FEE_RATE); - } - - /** - * Simulates a reachable provider, which successfully returns an API response - */ - public static FeeRateProvider buildDummyReachableMempoolFeeRateProvider(long feeRate) { - MempoolFeeRateProvider dummyProvider = new MempoolFeeRateProvider.First(env) { - @Override - protected FeeRate doGet() { - return new FeeRate("BTC", feeRate, MIN_FEE_RATE_FOR_WITHDRAWAL, Instant.now().getEpochSecond()); - } - }; - - // Initialize provider - dummyProvider.start(); - try { - sleep(1000); - } catch (InterruptedException e) { } - dummyProvider.stop(); - - return dummyProvider; - } - - /** - * Simulates an unreachable provider, which for whatever reason cannot deliver a - * response to the API. Reasons for that could be: host went offline, connection - * timeout, connection cannot be established (expired certificate), etc. - */ - public static FeeRateProvider buildDummyUnreachableMempoolFeeRateProvider() throws RestClientException { - MempoolFeeRateProvider dummyProvider = new MempoolFeeRateProvider.First(env) { - @Override - protected FeeRate doGet() { - throw new RestClientException("Simulating connection error when trying to reach API endpoint"); - } - }; - - // Initialize provider - dummyProvider.start(); - try { - sleep(1000); - } catch (InterruptedException e) { } - dummyProvider.stop(); - - return dummyProvider; - } -} diff --git a/pricenode/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java b/pricenode/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java deleted file mode 100644 index 642f3fe8566..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/ExchangeRateServiceTest.java +++ /dev/null @@ -1,414 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot; - -import bisq.core.locale.CurrencyUtil; - -import org.springframework.core.env.Environment; -import org.springframework.core.env.StandardEnvironment; - -import com.google.common.collect.Sets; - -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.RandomUtils; - -import java.time.Duration; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.OptionalDouble; -import java.util.Set; -import java.util.stream.Collectors; - -import org.slf4j.LoggerFactory; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import static java.lang.Thread.sleep; -import static java.util.Arrays.asList; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - -public class ExchangeRateServiceTest { - - /** - * Logback version of the Slf4j logger used by {@link ExchangeRateService}. This - * allows us to test if specific messages were logged. - * See https://stackoverflow.com/a/52229629 - */ - private static Logger exchangeRateServiceLogger; - private static final String LIST_APPENDER_NAME = "testListAppender"; - - @BeforeAll - static void setup() { - // Get the logger object for logs in ExchangeRateService - exchangeRateServiceLogger = (Logger) LoggerFactory.getLogger(ExchangeRateService.class); - - // Initiate and append a ListAppender, which allows us to programmatically inspect - // log messages - ListAppender listAppender = new ListAppender<>(); - listAppender.setName(LIST_APPENDER_NAME); - listAppender.start(); - exchangeRateServiceLogger.addAppender(listAppender); - } - - @Test - public void getAllMarketPrices_withNoExchangeRates_logs_Exception() { - int numberOfCurrencyPairsOnExchange = 0; - ExchangeRateProvider dummyProvider = buildDummyExchangeRateProvider(numberOfCurrencyPairsOnExchange); - ExchangeRateService service = new ExchangeRateService(Collections.singletonList(dummyProvider)); - - Map retrievedData = service.getAllMarketPrices(); - - doSanityChecksForRetrievedDataSingleProvider(retrievedData, dummyProvider, numberOfCurrencyPairsOnExchange); - - // No exchange rates provided by this exchange, two things should happen - // A) the timestamp should be set to 0 - // B) an error should be logged - - // A) Check that timestamp was set to 0 - // This allows Bisq clients to eventually disregard data from this provider - assertEquals(0L, retrievedData.get(dummyProvider.getPrefix() + "Ts")); - - // B) Check that an error is logged - // Log msg has the format: java.lang.IllegalStateException: No exchange rate data - // found for ExchangeName-JzfP1 - List logsList = ((ListAppender) exchangeRateServiceLogger.getAppender(LIST_APPENDER_NAME)).list; - assertEquals(Level.ERROR, logsList.get(0).getLevel()); - assertTrue(logsList.get(0).getMessage().endsWith("No exchange rate data found for " + dummyProvider.getName())); - } - - @Test - public void getAllMarketPrices_withSingleExchangeRate() { - int numberOfCurrencyPairsOnExchange = 1; - ExchangeRateProvider dummyProvider = buildDummyExchangeRateProvider(numberOfCurrencyPairsOnExchange); - ExchangeRateService service = new ExchangeRateService(Collections.singletonList(dummyProvider)); - - Map retrievedData = service.getAllMarketPrices(); - - doSanityChecksForRetrievedDataSingleProvider(retrievedData, dummyProvider, numberOfCurrencyPairsOnExchange); - - // One rate was provided by this provider, so the timestamp should not be 0 - assertNotEquals(0L, retrievedData.get(dummyProvider.getPrefix() + "Ts")); - } - - @Test - public void getAllMarketPrices_withMultipleProviders_differentCurrencyCodes() { - int numberOfCurrencyPairsOnExchange = 1; - ExchangeRateProvider dummyProvider1 = buildDummyExchangeRateProvider(numberOfCurrencyPairsOnExchange); - ExchangeRateProvider dummyProvider2 = buildDummyExchangeRateProvider(numberOfCurrencyPairsOnExchange); - ExchangeRateService service = new ExchangeRateService(asList(dummyProvider1, dummyProvider2)); - - Map retrievedData = service.getAllMarketPrices(); - - doSanityChecksForRetrievedDataMultipleProviders(retrievedData, asList(dummyProvider1, dummyProvider2)); - - // One rate was provided by each provider in this service, so the timestamp - // (for both providers) should not be 0 - assertNotEquals(0L, retrievedData.get(dummyProvider1.getPrefix() + "Ts")); - assertNotEquals(0L, retrievedData.get(dummyProvider2.getPrefix() + "Ts")); - } - - /** - * Tests the scenario when multiple providers have rates for the same currencies - */ - @Test - public void getAllMarketPrices_withMultipleProviders_overlappingCurrencyCodes() { - - // List of currencies for which multiple providers will have exchange rates - Set rateCurrencyCodes = Sets.newHashSet("CURRENCY-1", "CURRENCY-2", "CURRENCY-3"); - - // Create several dummy providers, each providing their own rates for the same set of currencies - ExchangeRateProvider dummyProvider1 = buildDummyExchangeRateProvider(rateCurrencyCodes); - ExchangeRateProvider dummyProvider2 = buildDummyExchangeRateProvider(rateCurrencyCodes); - - ExchangeRateService service = new ExchangeRateService(asList(dummyProvider1, dummyProvider2)); - - Map retrievedData = service.getAllMarketPrices(); - - doSanityChecksForRetrievedDataMultipleProviders(retrievedData, asList(dummyProvider1, dummyProvider2)); - - // At least one rate was provided by each provider in this service, so the - // timestamp (for both providers) should not be 0 - assertNotEquals(0L, retrievedData.get(dummyProvider1.getPrefix() + "Ts")); - assertNotEquals(0L, retrievedData.get(dummyProvider2.getPrefix() + "Ts")); - } - - /** - * Tests the scenario when currencies are excluded from the PriceNode feed via configuration settings - */ - @Test - public void getAllMarketPrices_withMultipleProviders_excludedCurrencyCodes() { - String excludedCcyString = "LBP,USD,EUR"; - String providerExcludedCcyString = "HUOBI:BRL,BINANCE:GBP,BINANCE:SEK"; - Environment mockedEnvironment = mock(Environment.class); - when(mockedEnvironment.getProperty(eq("bisq.price.fiatcurrency.excluded"), anyString())).thenReturn(excludedCcyString); - when(mockedEnvironment.getProperty(eq("bisq.price.fiatcurrency.excludedByProvider"), anyString())).thenReturn(providerExcludedCcyString); - - class MockedExchangeRateProvider extends ExchangeRateProvider { - MockedExchangeRateProvider() { - super(mockedEnvironment, "ExchangeName", "EXCH", Duration.ofDays(1)); - } - - @Override - public boolean isRunning() { - return true; - } - - @Override - public Set doGet() { - HashSet exchangeRates = new HashSet<>(); - // Simulate rates for all the supported ccys - for (String rateCurrencyCode : getSupportedFiatCurrencies()) { - exchangeRates.add(new ExchangeRate( - rateCurrencyCode, - RandomUtils.nextDouble(1, 1000), // random price - System.currentTimeMillis(), - getName())); // ExchangeRateProvider name - } - return exchangeRates; - } - } - - Logger exchangeRateProviderLogger; - String LIST_APPENDER_NAME2 = "testListAppender2"; - exchangeRateProviderLogger = (Logger) LoggerFactory.getLogger(MockedExchangeRateProvider.class); - ListAppender listAppender2 = new ListAppender<>(); - listAppender2.setName(LIST_APPENDER_NAME2); - listAppender2.start(); - exchangeRateProviderLogger.addAppender(listAppender2); - - // we request rates for all currencies, and check that: - // (a) the provider supplies more currency rates than the number of currencies we are trying to exclude (sanity test), - // (b) the number of missing currency rates equals the number of currencies we told PriceNode to exclude, - // (c) none of the rates supplied are for an excluded currency. - - Set excludedFiatCurrencies = new HashSet<>(asList(excludedCcyString.split(","))); - MockedExchangeRateProvider mockedExchangeRateProvider = new MockedExchangeRateProvider(); - Set exchangeRates = mockedExchangeRateProvider.doGet(); - assertTrue(exchangeRates.size() > excludedFiatCurrencies.size()); - int numSortedFiatCurrencies = CurrencyUtil.getAllSortedFiatCurrencies().size(); - int numCurrenciesFromProvider = mockedExchangeRateProvider.getSupportedFiatCurrencies().size(); - int missingCurrencyCount = numSortedFiatCurrencies - numCurrenciesFromProvider; - assertEquals(missingCurrencyCount, excludedFiatCurrencies.size()); - for (ExchangeRate exchangeRate : exchangeRates) { - assertFalse(excludedCcyString.contains(exchangeRate.getCurrency())); - } - List logsList = ((ListAppender) exchangeRateProviderLogger.getAppender(LIST_APPENDER_NAME2)).list; - assertEquals(3, logsList.size()); - assertEquals(Level.INFO, logsList.get(1).getLevel()); - assertTrue(logsList.get(0).getFormattedMessage().endsWith("will refresh every PT24H")); - assertTrue(logsList.get(1).getFormattedMessage().endsWith("fiat currencies excluded: [LBP, USD, EUR]")); - assertTrue(logsList.get(2).getFormattedMessage().endsWith("fiat currencies supported: " + numCurrenciesFromProvider)); - } - - /** - * Performs generic sanity checks on the response format and contents. - * - * @param retrievedData Response data retrieved from the {@link ExchangeRateService} - * @param provider {@link ExchangeRateProvider} available to the - * {@link ExchangeRateService} - * @param numberOfCurrencyPairsOnExchange Number of currency pairs this exchange was - * initiated with - */ - private void doSanityChecksForRetrievedDataSingleProvider(Map retrievedData, - ExchangeRateProvider provider, - int numberOfCurrencyPairsOnExchange) { - // Check response structure - doSanityChecksForRetrievedDataMultipleProviders(retrievedData, asList(provider)); - - // Check that the amount of provided exchange rates matches expected value - // For one provider, the amount of rates of that provider should be the total - // amount of rates in the response - List retrievedMarketPricesData = (List) retrievedData.get("data"); - assertEquals(numberOfCurrencyPairsOnExchange, retrievedMarketPricesData.size()); - } - - /** - * Performs generic sanity checks on the response format and contents. - * - * @param retrievedData Response data retrieved from the {@link ExchangeRateService} - * @param providers List of all {@link ExchangeRateProvider#getPrefix()} the - * {@link ExchangeRateService} uses - */ - private void doSanityChecksForRetrievedDataMultipleProviders(Map retrievedData, - List providers) { - // Check the correct amount of entries were present in the service response: - // The timestamp and the count fields are per provider, so N providers means N - // times those fields timestamp (x N) + count (x N) + price data (stored as a list - // under the key "data"). So expected size is Nx2 + 1. - int n = providers.size(); - assertEquals(n * 2 + 1, retrievedData.size()); - for (ExchangeRateProvider provider : providers) { - String providerPrefix = provider.getPrefix(); - assertNotNull(retrievedData.get(providerPrefix + "Ts")); - assertNotNull(retrievedData.get(providerPrefix + "Count")); - } - - // Check validity of the data field - List retrievedRates = (List) retrievedData.get("data"); - assertNotNull(retrievedRates); - - // It should contain no duplicate ExchangeRate objects - int uniqueRates = Sets.newHashSet(retrievedRates).size(); - int totalRates = retrievedRates.size(); - assertEquals(uniqueRates, totalRates, "Found duplicate rates in data field"); - - // There should be only one ExchangeRate per currency - // In other words, even if multiple providers return rates for the same currency, - // the ExchangeRateService should expose only one (aggregate) ExchangeRate for - // that currency - Map currencyCodeToExchangeRateFromService = retrievedRates.stream() - .collect(Collectors.toMap( - ExchangeRate::getCurrency, exchangeRate -> exchangeRate - )); - int uniqueCurrencyCodes = currencyCodeToExchangeRateFromService.keySet().size(); - assertEquals(uniqueCurrencyCodes, uniqueRates, "Found currency code with multiple exchange rates"); - - // Collect all ExchangeRates from all providers and group them by currency code - Map> currencyCodeToExchangeRatesFromProviders = new HashMap<>(); - for (ExchangeRateProvider p : providers) { - for (ExchangeRate exchangeRate : p.get()) { - String currencyCode = exchangeRate.getCurrency(); - if (currencyCodeToExchangeRatesFromProviders.containsKey(currencyCode)) { - List l = new ArrayList<>(currencyCodeToExchangeRatesFromProviders.get(currencyCode)); - l.add(exchangeRate); - currencyCodeToExchangeRatesFromProviders.put(currencyCode, l); - } else { - currencyCodeToExchangeRatesFromProviders.put(currencyCode, asList(exchangeRate)); - } - } - } - - // For each ExchangeRate which is covered by multiple providers, ensure the rate - // value is an average - currencyCodeToExchangeRatesFromProviders.forEach((currencyCode, exchangeRateList) -> { - ExchangeRate rateFromService = currencyCodeToExchangeRateFromService.get(currencyCode); - double priceFromService = rateFromService.getPrice(); - - OptionalDouble opt = exchangeRateList.stream().mapToDouble(ExchangeRate::getPrice).average(); - double priceAvgFromProviders = opt.getAsDouble(); - - // Ensure that the ExchangeRateService correctly aggregates exchange rates - // from multiple providers. If multiple providers contain rates for a - // currency, the service should return a single aggregate rate - // Expected value for aggregate rate = avg(provider rates) - // This formula works for any number of providers for a specific currency - assertEquals(priceFromService, priceAvgFromProviders, "Service returned incorrect aggregate rate"); - }); - } - - /** - * @param numberOfRatesAvailable Number of exchange rates this provider returns - * @return Dummy {@link ExchangeRateProvider} providing rates for - * "numberOfRatesAvailable" random currency codes - */ - private ExchangeRateProvider buildDummyExchangeRateProvider(int numberOfRatesAvailable) { - ExchangeRateProvider dummyProvider = new ExchangeRateProvider( - new StandardEnvironment(), - "ExchangeName-" + getRandomAlphaNumericString(5), - "EXCH-" + getRandomAlphaNumericString(3), - Duration.ofDays(1)) { - - @Override - public boolean isRunning() { - return true; - } - - @Override - protected Set doGet() { - HashSet exchangeRates = new HashSet<>(); - - // Simulate the required amount of rates - for (int i = 0; i < numberOfRatesAvailable; i++) { - exchangeRates.add(new ExchangeRate( - // random symbol, avoid duplicates - "DUM-" + getRandomAlphaNumericString(3), - RandomUtils.nextDouble(1, 1000), // random price - System.currentTimeMillis(), - getName())); // ExchangeRateProvider name - } - - return exchangeRates; - } - }; - - // Initialize provider - dummyProvider.start(); - try { - sleep(1000); - } catch (InterruptedException e) { } - dummyProvider.stop(); - - return dummyProvider; - } - - private ExchangeRateProvider buildDummyExchangeRateProvider(Set rateCurrencyCodes) { - ExchangeRateProvider dummyProvider = new ExchangeRateProvider( - new StandardEnvironment(), - "ExchangeName-" + getRandomAlphaNumericString(5), - "EXCH-" + getRandomAlphaNumericString(3), - Duration.ofDays(1)) { - - @Override - public boolean isRunning() { - return true; - } - - @Override - protected Set doGet() { - HashSet exchangeRates = new HashSet<>(); - - // Simulate the required amount of rates - for (String rateCurrencyCode : rateCurrencyCodes) { - exchangeRates.add(new ExchangeRate( - rateCurrencyCode, - RandomUtils.nextDouble(1, 1000), // random price - System.currentTimeMillis(), - getName())); // ExchangeRateProvider name - } - - return exchangeRates; - } - }; - - // Initialize provider - dummyProvider.start(); - try { - sleep(1000); - } catch (InterruptedException e) { } - dummyProvider.stop(); - - return dummyProvider; - } - - private static String getRandomAlphaNumericString(int length) { - return RandomStringUtils.random(length, true, true); - } -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/BTCMarketsTest.java b/pricenode/src/test/java/bisq/price/spot/providers/BTCMarketsTest.java deleted file mode 100644 index 511273aa3f2..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/BTCMarketsTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class BTCMarketsTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new BTCMarkets(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/BinanceTest.java b/pricenode/src/test/java/bisq/price/spot/providers/BinanceTest.java deleted file mode 100644 index e99520203f1..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/BinanceTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class BinanceTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Binance(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/BitfinexTest.java b/pricenode/src/test/java/bisq/price/spot/providers/BitfinexTest.java deleted file mode 100644 index e2dfa7019af..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/BitfinexTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class BitfinexTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Bitfinex(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/BitflyerTest.java b/pricenode/src/test/java/bisq/price/spot/providers/BitflyerTest.java deleted file mode 100644 index a722c5d1557..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/BitflyerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class BitflyerTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Bitflyer(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/BitstampTest.java b/pricenode/src/test/java/bisq/price/spot/providers/BitstampTest.java deleted file mode 100644 index 3c419745480..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/BitstampTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class BitstampTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Bitstamp(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/CoinGeckoTest.java b/pricenode/src/test/java/bisq/price/spot/providers/CoinGeckoTest.java deleted file mode 100644 index 86c765dcafc..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/CoinGeckoTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class CoinGeckoTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new CoinGecko(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/CoinbaseProTest.java b/pricenode/src/test/java/bisq/price/spot/providers/CoinbaseProTest.java deleted file mode 100644 index a3e12c8a159..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/CoinbaseProTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class CoinbaseProTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new CoinbasePro(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/CoinoneTest.java b/pricenode/src/test/java/bisq/price/spot/providers/CoinoneTest.java deleted file mode 100644 index 9832118ca00..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/CoinoneTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class CoinoneTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Coinone(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/IndependentReserveTest.java b/pricenode/src/test/java/bisq/price/spot/providers/IndependentReserveTest.java deleted file mode 100644 index 93a43a9cfdd..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/IndependentReserveTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class IndependentReserveTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new IndependentReserve(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/KrakenTest.java b/pricenode/src/test/java/bisq/price/spot/providers/KrakenTest.java deleted file mode 100644 index 8226c88defc..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/KrakenTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class KrakenTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Kraken(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/LunoTest.java b/pricenode/src/test/java/bisq/price/spot/providers/LunoTest.java deleted file mode 100644 index a04c9fb1c53..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/LunoTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class LunoTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Luno(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/MercadoBitcoinTest.java b/pricenode/src/test/java/bisq/price/spot/providers/MercadoBitcoinTest.java deleted file mode 100644 index 56715327d39..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/MercadoBitcoinTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class MercadoBitcoinTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new MercadoBitcoin(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/ParibuTest.java b/pricenode/src/test/java/bisq/price/spot/providers/ParibuTest.java deleted file mode 100644 index 3aac4b9e22e..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/ParibuTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class ParibuTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Paribu(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/PoloniexTest.java b/pricenode/src/test/java/bisq/price/spot/providers/PoloniexTest.java deleted file mode 100644 index de7711e92ce..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/PoloniexTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class PoloniexTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Poloniex(new StandardEnvironment())); - } - -} diff --git a/pricenode/src/test/java/bisq/price/spot/providers/QuoineTest.java b/pricenode/src/test/java/bisq/price/spot/providers/QuoineTest.java deleted file mode 100644 index 64653342c25..00000000000 --- a/pricenode/src/test/java/bisq/price/spot/providers/QuoineTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.price.spot.providers; - -import bisq.price.AbstractExchangeRateProviderTest; - -import org.springframework.core.env.StandardEnvironment; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Test; - -@Slf4j -public class QuoineTest extends AbstractExchangeRateProviderTest { - - @Test - public void doGet_successfulCall() { - doGet_successfulCall(new Quoine(new StandardEnvironment())); - } - -} diff --git a/pricenode/torrc b/pricenode/torrc deleted file mode 100644 index 3bfc6285eee..00000000000 --- a/pricenode/torrc +++ /dev/null @@ -1,2 +0,0 @@ -HiddenServiceDir build/tor-hidden-service -HiddenServicePort 80 127.0.0.1:8080 diff --git a/pricenode/uninstall_hsversion_debian.sh b/pricenode/uninstall_hsversion_debian.sh deleted file mode 100755 index 618e848953d..00000000000 --- a/pricenode/uninstall_hsversion_debian.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -set -e - -echo "[*] Network Size Monitoring removal script" - -##### change paths if necessary for your system - -ROOT_USER=root - -SCRAPER_HOME=/journalreader - -##### -echo "[*] Checking environment..." -if [ ! -f "${SCRAPER_HOME}/scraperscript_hsversion.sh" ]; then - echo 'There is nothing to be removed.' - echo 'Exiting...' - exit -fi - -echo "[*] Removing journal parser script" -sudo -H -i -u "${ROOT_USER}" rm "${SCRAPER_HOME}/scraperscript_hsversion.sh" - -echo "[*] Reverting collectd config" -sudo -H -i -u "${ROOT_USER}" sed -i '/.*/ {N;N; s/.*scraperscript_hsversion.sh.*<.Plugin>//g}' /etc/collectd/collectd.conf - -echo "[*] Restarting services" -sudo -H -i -u "${ROOT_USER}" systemctl restart collectd.service - -echo '[*] Done!' diff --git a/settings.gradle b/settings.gradle index 3bebfb81274..b9c33414f19 100644 --- a/settings.gradle +++ b/settings.gradle @@ -7,7 +7,6 @@ include 'cli' include 'daemon' include 'desktop' include 'monitor' -include 'pricenode' include 'seednode' include 'statsnode' include 'apitest' From f55ab5e56568b0c69ea45bceafd796ab93009780 Mon Sep 17 00:00:00 2001 From: HenrikJannsen Date: Thu, 24 Nov 2022 16:30:34 -0500 Subject: [PATCH 2/4] Remove monitor module It has been extracted to https://github.com/bisq-network/bisq-monitor Signed-off-by: HenrikJannsen --- .gitignore | 2 - Makefile | 1 - build.gradle | 47 --- monitor/README.md | 246 --------------- monitor/bisq-monitor.service | 16 - monitor/collectd.conf | 136 -------- monitor/install_collectd_debian.sh | 75 ----- monitor/nginx.conf | 28 -- .../main/java/bisq/monitor/AvailableTor.java | 52 ---- .../main/java/bisq/monitor/Configurable.java | 74 ----- .../src/main/java/bisq/monitor/Metric.java | 147 --------- .../src/main/java/bisq/monitor/Monitor.java | 182 ----------- .../main/java/bisq/monitor/OnionParser.java | 47 --- .../src/main/java/bisq/monitor/Reporter.java | 74 ----- .../java/bisq/monitor/StatisticsHelper.java | 70 ----- .../main/java/bisq/monitor/ThreadGate.java | 81 ----- .../java/bisq/monitor/metric/MarketStats.java | 100 ------ .../bisq/monitor/metric/P2PMarketStats.java | 282 ----------------- .../bisq/monitor/metric/P2PNetworkLoad.java | 245 --------------- .../bisq/monitor/metric/P2PRoundTripTime.java | 111 ------- .../monitor/metric/P2PSeedNodeSnapshot.java | 292 ------------------ .../metric/P2PSeedNodeSnapshotBase.java | 241 --------------- .../bisq/monitor/metric/PriceNodeStats.java | 165 ---------- .../metric/TorHiddenServiceStartupTime.java | 82 ----- .../bisq/monitor/metric/TorRoundTripTime.java | 92 ------ .../bisq/monitor/metric/TorStartupTime.java | 88 ------ .../monitor/reporter/ConsoleReporter.java | 70 ----- .../monitor/reporter/GraphiteReporter.java | 106 ------- monitor/src/main/resources/logback.xml | 12 - monitor/src/main/resources/metrics.properties | 79 ----- .../monitor/MonitorInfrastructureTests.java | 149 --------- .../bisq/monitor/P2PNetworkLoadTests.java | 118 ------- .../bisq/monitor/P2PRoundTripTimeTests.java | 136 -------- .../bisq/monitor/PriceNodeStatsTests.java | 115 ------- .../TorHiddenServiceStartupTimeTests.java | 115 ------- .../bisq/monitor/TorRoundTripTimeTests.java | 142 --------- .../bisq/monitor/TorStartupTimeTests.java | 92 ------ monitor/uninstall_collectd_debian.sh | 11 - settings.gradle | 1 - 39 files changed, 4122 deletions(-) delete mode 100644 monitor/README.md delete mode 100644 monitor/bisq-monitor.service delete mode 100644 monitor/collectd.conf delete mode 100755 monitor/install_collectd_debian.sh delete mode 100644 monitor/nginx.conf delete mode 100644 monitor/src/main/java/bisq/monitor/AvailableTor.java delete mode 100644 monitor/src/main/java/bisq/monitor/Configurable.java delete mode 100644 monitor/src/main/java/bisq/monitor/Metric.java delete mode 100644 monitor/src/main/java/bisq/monitor/Monitor.java delete mode 100644 monitor/src/main/java/bisq/monitor/OnionParser.java delete mode 100644 monitor/src/main/java/bisq/monitor/Reporter.java delete mode 100644 monitor/src/main/java/bisq/monitor/StatisticsHelper.java delete mode 100644 monitor/src/main/java/bisq/monitor/ThreadGate.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/MarketStats.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshotBase.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java delete mode 100644 monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java delete mode 100644 monitor/src/main/java/bisq/monitor/reporter/ConsoleReporter.java delete mode 100644 monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java delete mode 100644 monitor/src/main/resources/logback.xml delete mode 100644 monitor/src/main/resources/metrics.properties delete mode 100644 monitor/src/test/java/bisq/monitor/MonitorInfrastructureTests.java delete mode 100644 monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java delete mode 100644 monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java delete mode 100644 monitor/src/test/java/bisq/monitor/PriceNodeStatsTests.java delete mode 100644 monitor/src/test/java/bisq/monitor/TorHiddenServiceStartupTimeTests.java delete mode 100644 monitor/src/test/java/bisq/monitor/TorRoundTripTimeTests.java delete mode 100644 monitor/src/test/java/bisq/monitor/TorStartupTimeTests.java delete mode 100755 monitor/uninstall_collectd_debian.sh diff --git a/.gitignore b/.gitignore index b1cc84ff079..25ccd9afd15 100644 --- a/.gitignore +++ b/.gitignore @@ -30,8 +30,6 @@ desktop.ini *.class deploy */releases/* -/monitor/TorHiddenServiceStartupTimeTests/* -/monitor/monitor-tor/* .java-version .localnet /apitest/src/main/resources/dao-setup* diff --git a/Makefile b/Makefile index 58ff18290b2..1575383b664 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,6 @@ # # $ ls -1 bisq-* # bisq-desktop -# bisq-monitor # bisq-relay # bisq-seednode # bisq-statsnode diff --git a/build.gradle b/build.gradle index aebb42af476..5c4ae044e06 100644 --- a/build.gradle +++ b/build.gradle @@ -93,7 +93,6 @@ configure(subprojects) { configure([project(':cli'), project(':daemon'), project(':desktop'), - project(':monitor'), project(':seednode'), project(':statsnode'), project(':apitest')]) { @@ -506,52 +505,6 @@ configure(project(':desktop')) { } -configure(project(':monitor')) { - apply plugin: 'org.openjfx.javafxplugin' - - javafx { - version = "$javafxVersion" - modules = ['javafx.base'] - } - - mainClassName = 'bisq.monitor.Monitor' - - test { - useJUnitPlatform() - testLogging { - events "passed", "skipped", "failed" - } - } - - dependencies { - implementation project(':assets') - implementation project(':common') - implementation project(':core') - implementation project(':p2p') - annotationProcessor "org.projectlombok:lombok:$lombokVersion" - compileOnly "org.projectlombok:lombok:$lombokVersion" - implementation "ch.qos.logback:logback-classic:$logbackVersion" - implementation "ch.qos.logback:logback-core:$logbackVersion" - implementation "com.google.guava:guava:$guavaVersion" - implementation "org.slf4j:slf4j-api:$slf4jVersion" - implementation("com.github.bisq-network.netlayer:tor.external:$netlayerVersion") { - exclude(module: 'slf4j-api') - } - implementation("com.github.bisq-network.netlayer:tor.native:$netlayerVersion") { - exclude(module: 'slf4j-api') - } - implementation("com.google.inject:guice:$guiceVersion") { - exclude(module: 'guava') - } - testAnnotationProcessor "org.projectlombok:lombok:$lombokVersion" - testCompileOnly "org.projectlombok:lombok:$lombokVersion" - testImplementation "org.junit.jupiter:junit-jupiter-api:$jupiterVersion" - testImplementation "org.junit.jupiter:junit-jupiter-params:$jupiterVersion" - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$jupiterVersion") - } -} - - configure(project(':seednode')) { apply plugin: 'com.github.johnrengelman.shadow' diff --git a/monitor/README.md b/monitor/README.md deleted file mode 100644 index f2ed142b89a..00000000000 --- a/monitor/README.md +++ /dev/null @@ -1,246 +0,0 @@ -# Bisq Network Monitor Node - -The Bisq monitor node collects a set of metrics which are of interest to developers and users alike. These metrics are then made available through reporters. - -The *Settled* release features these metrics: -- Tor Startup Time: The time it takes to start Tor starting at a clean system, unpacking the shipped Tor binaries, firing up Tor until Tor is connected to the Tor network and ready to use. -- Tor Roundtrip Time: Given a bootstrapped Tor, the roundtrip time of connecting to a hidden service is measured. -- Tor Hidden Service Startup Time: Given a bootstrapped Tor, the time it takes to create and announce a freshly created hidden service. -- P2P Round Trip Time: A metric hitchhiking the Ping/Pong messages of the Keep-Alive-Mechanism to determine the Round Trip Time when issuing a Ping to a seed node. -- P2P Seed Node Message Snapshot: Get absolute number and constellation of messages a fresh Bisq client will get on startup. Also reports diffs between seed nodes on a per-message-type basis. -- P2P Network Load: listens to the P2P network and its broadcast messages. Reports every X seconds. -- P2P Market Statistics: a demonstration metric which extracts market information from broadcast messages. This demo implementation reports the number of open offers per market. - - -The *Settled* release features these reporters: -- A reporter that simply writes the findings to `System.err` -- A reporter that reports the findings to a Graphite/Carbon instance using the [plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol) - -## Configuration - -The *Bisq Network Monitor Node* is to be configured via a Java properties file. There is a default configuration file shipped with the monitor which reports to the one monitoring service currently up and running. - -If you want to tweak the configuration, you can pass the location of the file as command line parameter: - -``` -./bisq-monitor /path/to/your/config.properties -``` - -A sample configuration file looks like follows: - -``` -## System configuration - -# true overwrites the reporters picked by the developers (for debugging for example) (defaults to false) -System.useConsoleReporter=true - -# 0 -> BTC_MAINNET, 1 -> BTC_TESTNET (default) -System.baseCurrencyNetwork=0 - -## Each Metric is configured via a set of properties. -## -## The minimal set of properties required to run a Metric is: -## -## YourMetricName.enabled=true|false -## YourMetricName.run.interval=10 [seconds] - -#Edit and uncomment the lines below to your liking - -#TorStartupTime Metric -TorStartupTime.enabled=true -TorStartupTime.run.interval=100 -TorStartupTime.run.socksPort=90500 # so that there is no interference with a system Tor - -#TorRoundTripTime Metric -TorRoundTripTime.enabled=true -TorRoundTripTime.run.interval=100 -TorRoundTripTime.run.sampleSize=5 -TorRoundTripTime.run.hosts=http://expyuzz4wqqyqhjn.onion:80 # torproject.org hidden service - -#TorHiddenServiceStartupTime Metric -TorHiddenServiceStartupTime.enabled=true -TorHiddenServiceStartupTime.run.interval=100 -TorHiddenServiceStartupTime.run.localPort=90501 # so that there is no interference with a system Tor -TorHiddenServiceStartupTime.run.servicePort=90511 # so that there is no interference with a system Tor - -#P2PRoundTripTime Metric -P2PRoundTripTime.enabled=true -P2PRoundTripTime.run.interval=100 -P2PRoundTripTime.run.sampleSize=5 -P2PRoundTripTime.run.hosts=723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 -P2PRoundTripTime.run.torProxyPort=9060 - -#P2PNetworkLoad Metric -P2PNetworkLoad.enabled=true -P2PNetworkLoad.run.interval=100 -P2PNetworkLoad.run.torProxyPort=9061 -P2PNetworkLoad.run.historySize=500 - -#P2PNetworkMessageSnapshot Metric -P2PSeedNodeSnapshot.enabled=true -P2PSeedNodeSnapshot.run.interval=24 -P2PSeedNodeSnapshot.run.hosts=3f3cu2yw7u457ztq.onion:8000, 723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 -P2PSeedNodeSnapshot.run.torProxyPort=9062 - -#P2PMarketStats Metric -P2PMarketStats.enabled=true -P2PMarketStats.run.interval=37 -P2PMarketStats.run.hosts=ef5qnzx6znifo3df.onion:8000 -P2PMarketStats.run.torProxyPort=9063 - -#PriceNodeStats Metric -PriceNodeStats.enabled=true -PriceNodeStats.run.interval=42 -PriceNodeStats.run.hosts=http://5bmpx76qllutpcyp.onion, http://xc3nh4juf2hshy7e.onion, http://44mgyoe2b6oqiytt.onion, http://62nvujg5iou3vu3i.onion, http://ceaanhbvluug4we6.onion - -#MarketStats Metric -MarketStats.enabled=true -MarketStats.run.interval=191 - -## Reporters are configured via a set of properties as well. -## -## In contrast to Metrics, Reporters do not have a minimal set of properties. - -#GraphiteReporter -GraphiteReporter.serviceUrl=k6evlhg44acpchtc.onion:2003 - -``` - -## Run - -The distribution ships with a systemd .service file. Validate/change the executable/config paths within the shipped `bisq-monitor.service` file and copy/move the file to your systemd directory (something along `/usr/lib/systemd/system/`). Now you can control your *Monitor Node* via the usual systemd start/stop commands - -``` -systemctl start bisq-monitor.service -systemctl stop bisq-monitor.service -``` -and -``` -systemctl enable bisq-monitor.service -``` - -You can reload the configuration without restarting the service by using - -``` -systemctl reload bisq-monitor.service -``` - -Follow the logs created by the service by inspecting - -``` -journalctl --unit bisq-monitor --follow -``` - -# Monitoring Service - -A typical monitoring service consists of a [Graphite](https://graphiteapp.org/) and a [Grafana](https://grafana.com/) instance. -Both are available via Docker-containers. - -## Setting up Graphite - -### Install - -For a docker setup, use - -``` -docker run -d --name graphite --restart=always -p 2003:2003 -p 8080:8080 graphiteapp/graphite-statsd -``` - -- Port 2003 is used for the [plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol) mentioned above -- Port 8080 offers an API for user interfaces. - -more information can be found [here](https://graphite.readthedocs.io/en/latest/install.html) - -### Configuration - -For configuration, you must adapt the whisper database schema to suit your needs. First, stop your docker container by running - -``` -docker stop graphite -``` - -Find your config files within the `Source` directory stated in - -``` -docker inspect graphite | grep -C 2 graphite/conf\", -``` - -Edit `storage-schemas.conf` so that the frequency of your incoming data (configured in the monitor configs `interval`) is matched. For example, insert -``` -[bisq] -pattern = ^bisq.* -retentions = 10s:1h,5m:31d,30m:2y,1h:5y -``` -before the `[default...` blocks of the file. This basically says, that every incoming set of data reflects 5 minutes of the time series. Furthermore, every 30 minutes, the data is compressed and thus, takes less memory as it is kept for 2 years. - -Further, edit `storage-aggregation.conf` to configure how your data is compressed. For example, insert -``` -[bisq] -pattern=^bisq.* -xFilesFactor = 0 -aggregationMethod = average -``` -before the `[default...` blocks of the file. With this configuration, whenever data is aggregated, the `average` data is made available given that at least `0%` of the data points (i.e. floor(30 / 5 * 40%) = 2 data points) exist. Otherwise, the aggregated data is dropped. Since we start the first hour with a frequency of 10s but only supply data every 4 to 6 minutes, our aggregated values would get dropped. - -*Please note, that I have not been able to get the whole thing to work without the 10s:1h part yet* - -Finally, update the database. For doing that, go to the storage directory of graphite, the `Source` directory stated in -``` -docker inspect graphite | grep -C 2 graphite/conf\", -``` -Once there, you have two options: -- delete the whisper directory -``` -rm -r whisper -``` -- update the database by doing -``` -find ./ -type f -name '*.wsp' -exec whisper-resize.py --nobackup {} 10s:1h 5m:31d 30m:2y 1h:5y \; -``` - -and finally, restart your graphite container: -``` -docker start graphite -``` - - -Other than that, there is no further configuration necessary. However, you might change your iptables/firewalls to not let anyone access your Graphite instance from the outside. - -### Backup your data - -The metric data is kept in the `Source` directory stated in -``` -docker inspect graphite | grep -C 2 graphite/conf\", -``` -ready to be backed up regularly. - -## Setting up Grafana - -### Install - -For a docker setup, use - -``` -docker run -d --name=grafana -p 3000:3000 grafana/grafana -``` - -- Port 3000 offers the web interface - -more information can be found [here](https://grafana.com/grafana/download?platform=docker) - -### Configuration - -- Once you have Grafana up and running, go to the *Data Source* configuration tab. -- Once there click *Add data source* and select *Graphite*. -- In the HTTP section enter the IP address of your graphite docker container and the port `8080` (as we have configured before). E.g. `http://172.170.1:8080` -- Select `Server (default)` as an *Access* method and hit *Save & Test*. - -You should be all set. You can now proceed to add Dashboards, Panels and finally display the prettiest Graphs you can think of. -A working connection to Graphite should let you add your data series in a *Graph*s *Metrics* tab in a pretty intuitive way. - -- Optional: hide your Grafana instance behind a reverse proxy like nginx and add some TLS. -- Optional: make your Grafana instance accessible via a Tor hidden service. - -### Backup your data - -Grafana stores every dashboard as a JSON model. This model can be accessed (copied/restored) within the dashboard's settings and its *JSON Model* tab. Do with the data whatever you want. diff --git a/monitor/bisq-monitor.service b/monitor/bisq-monitor.service deleted file mode 100644 index 892c69ae53b..00000000000 --- a/monitor/bisq-monitor.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Bisq network monitor -After=network.target - -[Service] -WorkingDirectory=~ -Environment="JAVA_OPTS='-Xmx500M'" -ExecStart=/home/bisq/bisq/bisq-monitor /home/bisq/monitor.properties -ExecReload=/bin/kill -USR1 $MAINPID -Restart=on-failure - -User=bisq -Group=bisq - -[Install] -WantedBy=multi-user.target diff --git a/monitor/collectd.conf b/monitor/collectd.conf deleted file mode 100644 index f6106508b11..00000000000 --- a/monitor/collectd.conf +++ /dev/null @@ -1,136 +0,0 @@ -Hostname "__ONION_ADDRESS__" -Interval 30 - -LoadPlugin syslog - - LogLevel info - - -LoadPlugin cpu -LoadPlugin df -LoadPlugin disk -LoadPlugin fhcount -LoadPlugin interface -LoadPlugin java -LoadPlugin load -LoadPlugin memory -LoadPlugin processes -LoadPlugin swap -LoadPlugin write_graphite - - - ReportByCpu true - ValuesPercentage true - - - - MountPoint "/" - - - - Disk "/[hs]da/" - - - - ValuesAbsolute false - ValuesPercentage true - - - - Interface "eth0" - - - - JVMArg "-verbose:jni" - JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar" - - LoadPlugin "org.collectd.java.GenericJMX" - - # Generic heap/nonheap memory usage. - - ObjectName "java.lang:type=Memory" - #InstanceFrom "" - InstancePrefix "memory" - - # Creates four values: committed, init, max, used - - Type "memory" - #InstancePrefix "" - #InstanceFrom "" - Table true - Attribute "HeapMemoryUsage" - InstancePrefix "heap-" - - - # Creates four values: committed, init, max, used - - Type "memory" - #InstancePrefix "" - #InstanceFrom "" - Table true - Attribute "NonHeapMemoryUsage" - InstancePrefix "nonheap-" - - - - # Memory usage by memory pool. - - ObjectName "java.lang:type=MemoryPool,*" - InstancePrefix "memory_pool-" - InstanceFrom "name" - - - Type "memory" - #InstancePrefix "" - #InstanceFrom "" - Table true - Attribute "Usage" - - - - - ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:6969/jmxrmi" - Collect "memory_pool" - Collect "memory" - - - # See /usr/share/doc/collectd/examples/GenericJMX.conf - # for an example config. - - - -# -# ReportRelative true -# - -# -# ValuesAbsolute true -# ValuesPercentage false -# - -# -# Process "name" -# ProcessMatch "foobar" "/usr/bin/perl foobar\\.pl.*" -# - -# -# ReportByDevice false -# ReportBytes true -# - - - - Host "127.0.0.1" - Port "2003" - Protocol "tcp" - ReconnectInterval 0 - LogSendErrors false - Prefix "servers." - StoreRates true - AlwaysAppendDS false - EscapeCharacter "_" - SeparateInstances false - PreserveSeparator false - DropDuplicateFields false - - diff --git a/monitor/install_collectd_debian.sh b/monitor/install_collectd_debian.sh deleted file mode 100755 index 83bdf339db7..00000000000 --- a/monitor/install_collectd_debian.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -set -e - -echo "[*] Bisq Server Monitoring installation script" - -##### change paths if necessary for your system -BISQ_REPO_URL=https://raw.githubusercontent.com/bisq-network/bisq -BISQ_REPO_TAG=master -ROOT_USER=root -ROOT_GROUP=root -ROOT_HOME=~root -ROOT_PKG=(nginx collectd openssl) - -SYSTEMD_ENV_HOME=/etc/default - -##### - -echo "[*] Gathering information" -read -p "Please provide the onion address of your service (eg. 3f3cu2yw7u457ztq): " onionaddress - -echo "[*] Updating apt repo sources" -sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get update -q - -echo "[*] Upgrading OS packages" -sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get upgrade -qq -y - -echo "[*] Installing base packages" -sudo -H -i -u "${ROOT_USER}" DEBIAN_FRONTEND=noninteractive apt-get install -qq -y ${ROOT_PKG[@]} - -echo "[*] Preparing Bisq init script for monitoring" -# remove stuff it it is there already -for file in "${SYSTEMD_ENV_HOME}/bisq.env" "${SYSTEMD_ENV_HOME}/bisq-pricenode.env" -do - if [ -f "$file" ];then - sudo -H -i -u "${ROOT_USER}" sed -i -e 's/-Dcom.sun.management.jmxremote //g' -e 's/-Dcom.sun.management.jmxremote.local.only=true//g' -e 's/ -Dcom.sun.management.jmxremote.host=127.0.0.1//g' -e 's/ -Dcom.sun.management.jmxremote.port=6969//g' -e 's/ -Dcom.sun.management.jmxremote.rmi.port=6969//g' -e 's/ -Dcom.sun.management.jmxremote.ssl=false//g' -e 's/ -Dcom.sun.management.jmxremote.authenticate=false//g' "${file}" - sudo -H -i -u "${ROOT_USER}" sed -i -e '/JAVA_OPTS/ s/"$/ -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=true -Dcom.sun.management.jmxremote.host=127.0.0.1 -Dcom.sun.management.jmxremote.port=6969 -Dcom.sun.management.jmxremote.rmi.port=6969 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"/' "${file}" - fi -done - -echo "[*] Seeding entropy from /dev/urandom" -sudo -H -i -u "${ROOT_USER}" /bin/sh -c "head -1500 /dev/urandom > ${ROOT_HOME}/.rnd" -echo "[*] Installing Nginx config" -sudo -H -i -u "${ROOT_USER}" openssl req -x509 -nodes -newkey rsa:2048 -days 3000 -keyout /etc/nginx/cert.key -out /etc/nginx/cert.crt -subj="/O=Bisq/OU=Bisq Infrastructure/CN=$onionaddress" -curl -s "${BISQ_REPO_URL}/${BISQ_REPO_TAG}/monitor/nginx.conf" > /tmp/nginx.conf -sudo -H -i -u "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 /tmp/nginx.conf /etc/nginx/nginx.conf - -echo "[*] Installing collectd config" -curl -s "${BISQ_REPO_URL}/${BISQ_REPO_TAG}/monitor/collectd.conf" > /tmp/collectd.conf -sudo -H -i -u "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 /tmp/collectd.conf /etc/collectd/collectd.conf -sudo -H -i -u "${ROOT_USER}" sed -i -e "s/__ONION_ADDRESS__/$onionaddress/" /etc/collectd/collectd.conf - -echo "[*] Updating systemd daemon configuration" -sudo -H -i -u "${ROOT_USER}" systemctl daemon-reload -sudo -H -i -u "${ROOT_USER}" systemctl enable nginx.service -sudo -H -i -u "${ROOT_USER}" systemctl enable collectd.service - -echo "[*] Restarting services" -set +e -service bisq status >/dev/null 2>&1 -[ $? != 4 ] && sudo -H -i -u "${ROOT_USER}" systemctl restart bisq.service -service bisq-pricenode status >/dev/null 2>&1 -[ $? != 4 ] && sudo -H -i -u "${ROOT_USER}" systemctl restart bisq-pricenode.service -sudo -H -i -u "${ROOT_USER}" systemctl restart nginx.service -sudo -H -i -u "${ROOT_USER}" systemctl restart collectd.service - -echo '[*] Done!' - -echo ' ' -echo '[*] Report this certificate to the monitoring team!' -echo '----------------------------------------------------------------' -echo "Server: $onionaddress" -echo ' ' -cat /etc/nginx/cert.crt -echo '----------------------------------------------------------------' -echo ' ' diff --git a/monitor/nginx.conf b/monitor/nginx.conf deleted file mode 100644 index fc1a962f148..00000000000 --- a/monitor/nginx.conf +++ /dev/null @@ -1,28 +0,0 @@ -load_module /usr/lib/nginx/modules/ngx_stream_module.so; - -worker_processes 1; - -events { - worker_connections 1024; -} - -stream { - - log_format basic '$remote_addr [$time_local] ' - '$protocol Status $status Sent $bytes_sent Received $bytes_received ' - 'Time $session_time'; - - error_log syslog:server=unix:/dev/log; - access_log syslog:server=unix:/dev/log basic; - - server { - listen 127.0.0.1:2003; - proxy_pass monitor.bisq.network:2002; - proxy_ssl on; - - proxy_ssl_certificate /etc/nginx/cert.crt; - proxy_ssl_certificate_key /etc/nginx/cert.key; - - proxy_ssl_session_reuse on; - } -} diff --git a/monitor/src/main/java/bisq/monitor/AvailableTor.java b/monitor/src/main/java/bisq/monitor/AvailableTor.java deleted file mode 100644 index 0c79ec48e39..00000000000 --- a/monitor/src/main/java/bisq/monitor/AvailableTor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.network.p2p.network.TorMode; - -import org.berndpruenster.netlayer.tor.Tor; - -import java.io.File; - -/** - * This class uses an already defined Tor via Tor.getDefault() - * - * @author Florian Reimair - * - */ -public class AvailableTor extends TorMode { - - private final String hiddenServiceDirectory; - - public AvailableTor(File torWorkingDirectory, String hiddenServiceDirectory) { - super(torWorkingDirectory); - - this.hiddenServiceDirectory = hiddenServiceDirectory; - } - - @Override - public Tor getTor() { - return Tor.getDefault(); - } - - @Override - public String getHiddenServiceDirectory() { - return hiddenServiceDirectory; - } - -} diff --git a/monitor/src/main/java/bisq/monitor/Configurable.java b/monitor/src/main/java/bisq/monitor/Configurable.java deleted file mode 100644 index 6280f02255b..00000000000 --- a/monitor/src/main/java/bisq/monitor/Configurable.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import java.util.Properties; - -/** - * Does some pre-computation for a configurable class. - * - * @author Florian Reimair - */ -public abstract class Configurable { - - protected Properties configuration = new Properties(); - - private String name; - - /** - * Filters all java properties starting with {@link Configurable#getName()} of - * the class and makes them available. Does NOT parse the content of - * the properties! - *

- * For example, if the implementing class sets its name (using - * {@link Configurable#setName(String)}) to MyName, the list of - * properties is scanned for properties starting with MyName. - * Matching lines are made available to the class without the prefix. For - * example, a property MyName.answer=42 is made available as - * configuration.getProperty("answer") resulting in - * 42. - * - * @param properties a set of configuration properties - */ - public void configure(final Properties properties) { - // only configure the Properties which belong to us - final Properties myProperties = new Properties(); - properties.forEach((k, v) -> { - String key = (String) k; - if (key.startsWith(getName())) - myProperties.put(key.substring(key.indexOf(".") + 1), v); - }); - - // configure all properties that belong to us - this.configuration = myProperties; - } - - protected String getName() { - return name; - } - - /** - * Set the name used to filter through configuration properties. See - * {@link Configurable#configure(Properties)}. - * - * @param name the name of the configurable - */ - protected void setName(String name) { - this.name = name; - } -} diff --git a/monitor/src/main/java/bisq/monitor/Metric.java b/monitor/src/main/java/bisq/monitor/Metric.java deleted file mode 100644 index 08598e1e9cf..00000000000 --- a/monitor/src/main/java/bisq/monitor/Metric.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.common.app.Version; -import bisq.common.util.Utilities; - -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -import static bisq.common.config.Config.BASE_CURRENCY_NETWORK; - -/** - * Starts a Metric (in its own {@link Thread}), manages its properties and shuts - * it down gracefully. Furthermore, configuration updates and execution are done - * in a thread-save manner. Implementing classes only have to implement the - * {@link Metric#execute()} method. - * - * @author Florian Reimair - */ -@Slf4j -public abstract class Metric extends Configurable implements Runnable { - - private static final String INTERVAL = "run.interval"; - private static ScheduledExecutorService executor; - protected final Reporter reporter; - private ScheduledFuture scheduler; - - /** - * disable execution - */ - private void disable() { - if (scheduler != null) - scheduler.cancel(false); - } - - /** - * enable execution - */ - private void enable() { - scheduler = executor.scheduleWithFixedDelay(this, new Random().nextInt(60), - Long.parseLong(configuration.getProperty(INTERVAL)), TimeUnit.SECONDS); - } - - /** - * Constructor. - */ - protected Metric(Reporter reporter) { - - this.reporter = reporter; - - setName(this.getClass().getSimpleName()); - - if (executor == null) { - executor = new ScheduledThreadPoolExecutor(6); - } - } - - boolean enabled() { - if (scheduler != null) - return !scheduler.isCancelled(); - else - return false; - } - - @Override - public void configure(final Properties properties) { - synchronized (this) { - log.info("{} (re)loading config...", getName()); - super.configure(properties); - reporter.configure(properties); - - Version.setBaseCryptoNetworkId(Integer.parseInt(properties.getProperty("System." + BASE_CURRENCY_NETWORK, "1"))); // defaults to BTC_TESTNET - - // decide whether to enable or disable the task - if (configuration.isEmpty() || !configuration.getProperty("enabled", "false").equals("true") - || !configuration.containsKey(INTERVAL)) { - disable(); - - // some informative log output - if (configuration.isEmpty()) - log.error("{} is not configured at all. Will not run.", getName()); - else if (!configuration.getProperty("enabled", "false").equals("true")) - log.info("{} is deactivated. Will not run.", getName()); - else if (!configuration.containsKey(INTERVAL)) - log.error("{} is missing mandatory '" + INTERVAL + "' property. Will not run.", getName()); - else - log.error("{} is mis-configured. Will not run.", getName()); - } else if (!enabled() && configuration.getProperty("enabled", "false").equals("true")) { - // check if this Metric got activated after being disabled. - // if so, resume execution - enable(); - log.info("{} got activated. Starting up.", getName()); - } - } - } - - @Override - public void run() { - try { - Thread.currentThread().setName("Metric: " + getName()); - - // execute all the things - synchronized (this) { - log.info("{} started", getName()); - execute(); - log.info("{} done", getName()); - } - } catch (Throwable e) { - log.error("A metric misbehaved!", e); - } - } - - /** - * Gets scheduled repeatedly. - */ - protected abstract void execute(); - - /** - * initiate an orderly shutdown on all metrics. Blocks until all metrics are - * shut down or after one minute. - */ - public static void haltAllMetrics() { - Utilities.shutdownAndAwaitTermination(executor, 2, TimeUnit.MINUTES); - } -} diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java deleted file mode 100644 index ea6b0db3619..00000000000 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.metric.MarketStats; -import bisq.monitor.metric.P2PMarketStats; -import bisq.monitor.metric.P2PNetworkLoad; -import bisq.monitor.metric.P2PRoundTripTime; -import bisq.monitor.metric.P2PSeedNodeSnapshot; -import bisq.monitor.metric.PriceNodeStats; -import bisq.monitor.metric.TorHiddenServiceStartupTime; -import bisq.monitor.metric.TorRoundTripTime; -import bisq.monitor.metric.TorStartupTime; -import bisq.monitor.reporter.ConsoleReporter; -import bisq.monitor.reporter.GraphiteReporter; - -import bisq.common.app.Capabilities; -import bisq.common.app.Capability; - -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; - -import java.io.File; -import java.io.FileInputStream; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import lombok.extern.slf4j.Slf4j; - -import static com.google.common.base.Preconditions.checkNotNull; - - - -import sun.misc.Signal; - -/** - * Monitor executable for the Bisq network. - * - * @author Florian Reimair - */ -@Slf4j -public class Monitor { - - public static final File TOR_WORKING_DIR = new File("monitor/work/monitor-tor"); - private static String[] args = {}; - - public static void main(String[] args) throws Throwable { - Monitor.args = args; - new Monitor().start(); - } - - /** - * A list of all active {@link Metric}s - */ - private final List metrics = new ArrayList<>(); - - /** - * Starts up all configured Metrics. - * - * @throws Throwable in case something goes wrong - */ - private void start() throws Throwable { - - // start Tor - Tor.setDefault(new NativeTor(TOR_WORKING_DIR, null, null, false)); - - //noinspection deprecation,deprecation,deprecation,deprecation,deprecation,deprecation,deprecation,deprecation - Capabilities.app.addAll(Capability.TRADE_STATISTICS, - Capability.TRADE_STATISTICS_2, - Capability.ACCOUNT_AGE_WITNESS, - Capability.ACK_MSG, - Capability.PROPOSAL, - Capability.BLIND_VOTE, - Capability.DAO_STATE, - Capability.BUNDLE_OF_ENVELOPES, - Capability.REFUND_AGENT, - Capability.MEDIATION, - Capability.TRADE_STATISTICS_3); - - // assemble Metrics - // - create reporters - Reporter graphiteReporter = new GraphiteReporter(); - - // only use ConsoleReporter if requested (for debugging for example) - Properties properties = getProperties(); - if ("true".equals(properties.getProperty("System.useConsoleReporter", "false"))) - graphiteReporter = new ConsoleReporter(); - - // - add available metrics with their reporters - metrics.add(new TorStartupTime(graphiteReporter)); - metrics.add(new TorRoundTripTime(graphiteReporter)); - metrics.add(new TorHiddenServiceStartupTime(graphiteReporter)); - metrics.add(new P2PRoundTripTime(graphiteReporter)); - metrics.add(new P2PNetworkLoad(graphiteReporter)); - metrics.add(new P2PSeedNodeSnapshot(graphiteReporter)); - metrics.add(new P2PMarketStats(graphiteReporter)); - metrics.add(new PriceNodeStats(graphiteReporter)); - metrics.add(new MarketStats(graphiteReporter)); - - // prepare configuration reload - // Note that this is most likely only work on Linux - Signal.handle(new Signal("USR1"), signal -> { - try { - configure(); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - }); - - // configure Metrics - // - which also starts the metrics if appropriate - configure(); - - // exit Metrics gracefully on shutdown - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - // set the name of the Thread for debugging purposes - log.info("system shutdown initiated"); - - log.info("shutting down active metrics..."); - Metric.haltAllMetrics(); - - try { - log.info("shutting down tor..."); - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - tor.shutdown(); - } catch (Throwable ignore) { - } - - log.info("system halt"); - }, "Monitor Shutdown Hook ") - ); - } - - /** - * Reload the configuration from disk. - * - * @throws Exception if something goes wrong - */ - private void configure() throws Exception { - Properties properties = getProperties(); - for (Metric current : metrics) - current.configure(properties); - } - - /** - * Overloads a default set of properties with a file if given - * - * @return a set of properties - * @throws Exception in case something goes wrong - */ - private Properties getProperties() throws Exception { - Properties result = new Properties(); - - // if we have a config file load the config file, else, load the default config - // from the resources - if (args.length > 0) - result.load(new FileInputStream(args[0])); - else - result.load(Monitor.class.getClassLoader().getResourceAsStream("metrics.properties")); - - return result; - } -} diff --git a/monitor/src/main/java/bisq/monitor/OnionParser.java b/monitor/src/main/java/bisq/monitor/OnionParser.java deleted file mode 100644 index 58f186f068a..00000000000 --- a/monitor/src/main/java/bisq/monitor/OnionParser.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.network.p2p.NodeAddress; - -import java.net.MalformedURLException; -import java.net.URL; - -/** - * Helper for parsing and pretty printing onion addresses. - * - * @author Florian Reimair - */ -public class OnionParser { - - public static NodeAddress getNodeAddress(final String current) throws MalformedURLException { - String nodeAddress = current.trim(); - if (!nodeAddress.startsWith("http://")) - nodeAddress = "http://" + nodeAddress; - URL tmp = new URL(nodeAddress); - return new NodeAddress(tmp.getHost(), tmp.getPort() > 0 ? tmp.getPort() : 80); - } - - public static String prettyPrint(final NodeAddress host) { - return host.getHostNameWithoutPostFix(); - } - - public static String prettyPrint(String host) throws MalformedURLException { - return prettyPrint(getNodeAddress(host)); - } -} diff --git a/monitor/src/main/java/bisq/monitor/Reporter.java b/monitor/src/main/java/bisq/monitor/Reporter.java deleted file mode 100644 index 5c0202e7f38..00000000000 --- a/monitor/src/main/java/bisq/monitor/Reporter.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import java.util.Map; - -/** - * Reports findings to a specific service/file/place using the proper means to - * do so. - * - * @author Florian Reimair - */ -public abstract class Reporter extends Configurable { - - protected Reporter() { - setName(this.getClass().getSimpleName()); - } - - /** - * Report our findings. - * - * @param value the value to report - */ - public abstract void report(long value); - - /** - * Report our findings - * - * @param value the value to report - * @param prefix a common prefix to be included in the tag name - */ - public abstract void report(long value, String prefix); - - /** - * Report our findings. - * - * @param values Map - */ - public abstract void report(Map values); - - /** - * Report our findings. - * - * @param values Map - * @param prefix for example "torStartupTime" - */ - public abstract void report(Map values, String prefix); - - /** - * Report our findings one by one. - * - * @param key the metric name - * @param value the value to report - * @param timestamp a unix timestamp in milliseconds - * @param prefix for example "torStartupTime" - */ - public abstract void report(String key, String value, String timestamp, String prefix); - -} diff --git a/monitor/src/main/java/bisq/monitor/StatisticsHelper.java b/monitor/src/main/java/bisq/monitor/StatisticsHelper.java deleted file mode 100644 index c73c904ff21..00000000000 --- a/monitor/src/main/java/bisq/monitor/StatisticsHelper.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.LongSummaryStatistics; -import java.util.Map; - -/** - * Calculates average, max, min, p25, p50, p75 off of a list of samples and - * throws in the sample size for good measure. - * - * @author Florian Reimair - */ -public class StatisticsHelper { - - public static Map process(Collection input) { - - List samples = new ArrayList<>(input); - - // aftermath - Collections.sort(samples); - - // - average, max, min , sample size - LongSummaryStatistics statistics = samples.stream().mapToLong(val -> val).summaryStatistics(); - - Map results = new HashMap<>(); - results.put("average", String.valueOf(Math.round(statistics.getAverage()))); - results.put("max", String.valueOf(statistics.getMax())); - results.put("min", String.valueOf(statistics.getMin())); - results.put("sampleSize", String.valueOf(statistics.getCount())); - - // - p25, median, p75 - Integer[] percentiles = new Integer[] { 25, 50, 75 }; - for (Integer percentile : percentiles) { - double rank = statistics.getCount() * percentile / 100.0; - Long percentileValue; - if (samples.size() <= rank + 1) - percentileValue = samples.get(samples.size() - 1); - else if (Math.floor(rank) == rank) - percentileValue = samples.get((int) rank); - else - percentileValue = Math.round(samples.get((int) Math.floor(rank)) - + (samples.get((int) (Math.floor(rank) + 1)) - samples.get((int) Math.floor(rank))) - / (rank - Math.floor(rank))); - results.put("p" + percentile, String.valueOf(percentileValue)); - } - - return results; - } -} diff --git a/monitor/src/main/java/bisq/monitor/ThreadGate.java b/monitor/src/main/java/bisq/monitor/ThreadGate.java deleted file mode 100644 index 60fa1586e3b..00000000000 --- a/monitor/src/main/java/bisq/monitor/ThreadGate.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -/** - * Gate pattern to help with thread synchronization - * - * @author Florian Reimair - */ -@Slf4j -public class ThreadGate { - - private CountDownLatch lock = new CountDownLatch(0); - - /** - * Make everyone wait until the gate is open again. - */ - public void engage() { - lock = new CountDownLatch(1); - } - - /** - * Make everyone wait until the gate is open again. - * - * @param numberOfLocks how often the gate has to be unlocked until the gate - * opens. - */ - public void engage(int numberOfLocks) { - lock = new CountDownLatch(numberOfLocks); - } - - /** - * Wait for the gate to be opened. Blocks until the gate is open again. Returns - * immediately if the gate is already open. - */ - public synchronized void await() { - while (lock.getCount() > 0) - try { - if (!lock.await(60, TimeUnit.SECONDS)) { - log.warn("timeout occurred!"); - break; // break the loop - } - } catch (InterruptedException ignore) { - } - } - - /** - * Open the gate and let everyone proceed with their execution. - */ - public void proceed() { - lock.countDown(); - } - - /** - * Open the gate with no regards on how many locks are still in place. - */ - public void unlock() { - while (lock.getCount() > 0) - lock.countDown(); - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/MarketStats.java b/monitor/src/main/java/bisq/monitor/metric/MarketStats.java deleted file mode 100644 index f54c862446c..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/MarketStats.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * This file is part of Bisq. - * - * bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.Metric; -import bisq.monitor.Reporter; - -import java.net.URL; -import java.net.URLConnection; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import lombok.extern.slf4j.Slf4j; - -/** - * Uses the markets API to retrieve market volume data. - * - * @author Florian Reimair - * - */ -@Slf4j -public class MarketStats extends Metric { - private static final String MARKETS_BISQ_NETWORK = "https://markets.bisq.network"; - // poor mans JSON parser - private final Pattern marketPattern = Pattern.compile("\"market\" ?: ?\"([a-z_]+)\""); - private final Pattern amountPattern = Pattern.compile("\"amount\" ?: ?\"([\\d\\.]+)\""); - private final Pattern volumePattern = Pattern.compile("\"volume\" ?: ?\"([\\d\\.]+)\""); - private final Pattern timestampPattern = Pattern.compile("\"trade_date\" ?: ?([\\d]+)"); - - private Long lastRun = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(15)); - - public MarketStats(Reporter reporter) { - super(reporter); - } - - @Override - protected void execute() { - try { - // for each configured host - Map result = new HashMap<>(); - - // assemble query - long now = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); - String query = "/api/trades?format=json&market=all×tamp_from=" + lastRun + "×tamp_to=" + now; - lastRun = now; // thought about adding 1 second but what if a trade is done exactly in this one second? - - // connect - URLConnection connection = new URL(MARKETS_BISQ_NETWORK + query).openConnection(); - - // prepare to receive data - BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); - - String line, all = ""; - while ((line = in.readLine()) != null) - all += ' ' + line; - in.close(); - - Arrays.stream(all.substring(0, all.length() - 2).split("}")).forEach(trade -> { - Matcher market = marketPattern.matcher(trade); - Matcher amount = amountPattern.matcher(trade); - Matcher timestamp = timestampPattern.matcher(trade); - market.find(); - if (market.group(1).endsWith("btc")) { - amount = volumePattern.matcher(trade); - } - amount.find(); - timestamp.find(); - reporter.report("volume." + market.group(1), amount.group(1), timestamp.group(1), getName()); - }); - } catch (IllegalStateException ignore) { - // no match found - } catch (IOException e) { - e.printStackTrace(); - } - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java deleted file mode 100644 index 368e9cb03c1..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * This file is part of Bisq. - * - * bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.Reporter; - -import bisq.core.offer.OfferUtil; -import bisq.core.offer.bisq_v1.OfferPayload; - -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; -import bisq.network.p2p.storage.payload.ProtectedStorageEntry; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import bisq.common.proto.network.NetworkEnvelope; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Demo Stats metric derived from the OfferPayload messages we get from the seed nodes - * - * @author Florian Reimair - */ -@Slf4j -public class P2PMarketStats extends P2PSeedNodeSnapshotBase { - final Map> versionBucketsPerHost = new ConcurrentHashMap<>(); - final Map> offerVolumeBucketsPerHost = new ConcurrentHashMap<>(); - final Map>> offerVolumeDistributionBucketsPerHost = new ConcurrentHashMap<>(); - final Map>> offersPerTraderBucketsPerHost = new ConcurrentHashMap<>(); - final Map>> volumePerTraderBucketsPerHost = new ConcurrentHashMap<>(); - - /** - * Efficient way to aggregate numbers. - */ - private static class Aggregator { - private long value = 0; - - synchronized long value() { - return value; - } - - synchronized void increment() { - value++; - } - - synchronized void add(long amount) { - value += amount; - } - } - - private abstract static class OfferStatistics extends Statistics { - @Override - public synchronized void log(Object message) { - if (message instanceof OfferPayload) { - OfferPayload currentMessage = (OfferPayload) message; - // For logging different data types - String market = currentMessage.getDirection() + "." + currentMessage.getBaseCurrencyCode() + "_" + currentMessage.getCounterCurrencyCode(); - - process(market, currentMessage); - } - } - - abstract void process(String market, OfferPayload currentMessage); - } - - private class OfferCountStatistics extends OfferStatistics { - - @Override - void process(String market, OfferPayload currentMessage) { - buckets.putIfAbsent(market, new Aggregator()); - buckets.get(market).increment(); - } - } - - private class OfferVolumeStatistics extends OfferStatistics { - - @Override - void process(String market, OfferPayload currentMessage) { - buckets.putIfAbsent(market, new Aggregator()); - buckets.get(market).add(currentMessage.getAmount()); - } - } - - private class OfferVolumeDistributionStatistics extends OfferStatistics> { - - @Override - void process(String market, OfferPayload currentMessage) { - buckets.putIfAbsent(market, new ArrayList<>()); - buckets.get(market).add(currentMessage.getAmount()); - } - } - - private class OffersPerTraderStatistics extends OfferStatistics> { - - @Override - void process(String market, OfferPayload currentMessage) { - buckets.putIfAbsent(market, new HashMap<>()); - buckets.get(market).putIfAbsent(currentMessage.getOwnerNodeAddress(), new Aggregator()); - buckets.get(market).get(currentMessage.getOwnerNodeAddress()).increment(); - } - } - - private class VolumePerTraderStatistics extends OfferStatistics> { - - @Override - void process(String market, OfferPayload currentMessage) { - buckets.putIfAbsent(market, new HashMap<>()); - buckets.get(market).putIfAbsent(currentMessage.getOwnerNodeAddress(), new Aggregator()); - buckets.get(market).get(currentMessage.getOwnerNodeAddress()).add(currentMessage.getAmount()); - } - } - - private class VersionsStatistics extends Statistics { - - @Override - public void log(Object message) { - - if (message instanceof OfferPayload) { - OfferPayload offerPayload = (OfferPayload) message; - String version = "v" + OfferUtil.getVersionFromId(offerPayload.getId()); - buckets.putIfAbsent(version, new Aggregator()); - buckets.get(version).increment(); - } - } - } - - public P2PMarketStats(Reporter graphiteReporter) { - super(graphiteReporter); - } - - @Override - protected List getRequests() { - List result = new ArrayList<>(); - - Random random = new Random(); - result.add(new PreliminaryGetDataRequest(random.nextInt(), hashes)); - - return result; - } - - protected void createHistogram(List input, String market, Map report) { - int numberOfBins = 5; - - // - get biggest offer - double max = input.stream().max(Long::compareTo).map(value -> value * 1.01).orElse(0.0); - - // - create histogram - input.stream().collect( - Collectors.groupingBy(aLong -> aLong == max ? numberOfBins - 1 : (int) Math.floor(aLong / (max / numberOfBins)), Collectors.counting())). - forEach((integer, integer2) -> report.put(market + ".bin_" + integer, String.valueOf(integer2))); - - report.put(market + ".number_of_bins", String.valueOf(numberOfBins)); - report.put(market + ".max", String.valueOf((int) max)); - } - - @Override - protected void report() { - Map report = new HashMap<>(); - bucketsPerHost.values().stream().findFirst().ifPresent(nodeAddressStatisticsEntry -> nodeAddressStatisticsEntry.values().forEach((market, numberOfOffers) -> report.put(market, String.valueOf(((Aggregator) numberOfOffers).value())))); - reporter.report(report, getName() + ".offerCount"); - - // do offerbook volume statistics - report.clear(); - offerVolumeBucketsPerHost.values().stream().findFirst().ifPresent(aggregatorStatistics -> aggregatorStatistics.values().forEach((market, numberOfOffers) -> report.put(market, String.valueOf(numberOfOffers.value())))); - reporter.report(report, getName() + ".volume"); - - // do the offer vs volume histogram - report.clear(); - // - get a data set - offerVolumeDistributionBucketsPerHost.values().stream().findFirst().ifPresent(listStatistics -> listStatistics.values().forEach((market, offers) -> { - createHistogram(offers, market, report); - })); - reporter.report(report, getName() + ".volume-per-offer-distribution"); - - // do offers per trader - report.clear(); - // - get a data set - offersPerTraderBucketsPerHost.values().stream().findFirst().ifPresent(mapStatistics -> mapStatistics.values().forEach((market, stuff) -> { - List offerPerTrader = stuff.values().stream().map(Aggregator::value).collect(Collectors.toList()); - - createHistogram(offerPerTrader, market, report); - })); - reporter.report(report, getName() + ".traders_by_number_of_offers"); - - // do volume per trader - report.clear(); - // - get a data set - volumePerTraderBucketsPerHost.values().stream().findFirst().ifPresent(mapStatistics -> mapStatistics.values().forEach((market, stuff) -> { - List volumePerTrader = stuff.values().stream().map(Aggregator::value).collect(Collectors.toList()); - - createHistogram(volumePerTrader, market, report); - })); - reporter.report(report, getName() + ".traders_by_volume"); - - // do version statistics - report.clear(); - Optional> optionalStatistics = versionBucketsPerHost.values().stream().findAny(); - optionalStatistics.ifPresent(aggregatorStatistics -> aggregatorStatistics.values() - .forEach((version, numberOfOccurrences) -> report.put(version, String.valueOf(numberOfOccurrences.value())))); - reporter.report(report, "versions"); - } - - protected boolean treatMessage(NetworkEnvelope networkEnvelope, Connection connection) { - checkNotNull(connection.getPeersNodeAddressProperty(), - "although the property is nullable, we need it to not be null"); - - if (networkEnvelope instanceof GetDataResponse) { - - Statistics offerCount = new OfferCountStatistics(); - Statistics offerVolume = new OfferVolumeStatistics(); - Statistics offerVolumeDistribution = new OfferVolumeDistributionStatistics(); - Statistics offersPerTrader = new OffersPerTraderStatistics(); - Statistics volumePerTrader = new VolumePerTraderStatistics(); - Statistics versions = new VersionsStatistics(); - - GetDataResponse dataResponse = (GetDataResponse) networkEnvelope; - final Set dataSet = dataResponse.getDataSet(); - dataSet.forEach(e -> { - final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); - if (protectedStoragePayload == null) { - log.warn("StoragePayload was null: {}", networkEnvelope.toString()); - return; - } - - offerCount.log(protectedStoragePayload); - offerVolume.log(protectedStoragePayload); - offerVolumeDistribution.log(protectedStoragePayload); - offersPerTrader.log(protectedStoragePayload); - volumePerTrader.log(protectedStoragePayload); - versions.log(protectedStoragePayload); - }); - - dataResponse.getPersistableNetworkPayloadSet().forEach(persistableNetworkPayload -> { - // memorize message hashes - //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; - //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); - - //hashes.add(bytes); - - hashes.add(persistableNetworkPayload.getHash()); - }); - - bucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), offerCount); - offerVolumeBucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), offerVolume); - offerVolumeDistributionBucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), offerVolumeDistribution); - offersPerTraderBucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), offersPerTrader); - volumePerTraderBucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), volumePerTrader); - versionBucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), versions); - return true; - } - return false; - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java deleted file mode 100644 index ff4bb78b6f1..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.AvailableTor; -import bisq.monitor.Metric; -import bisq.monitor.Monitor; -import bisq.monitor.Reporter; -import bisq.monitor.ThreadGate; - -import bisq.core.network.p2p.seed.DefaultSeedNodeRepository; -import bisq.core.proto.network.CoreNetworkProtoResolver; -import bisq.core.proto.persistable.CorePersistenceProtoResolver; - -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.network.MessageListener; -import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.network.SetupListener; -import bisq.network.p2p.network.TorNetworkNode; -import bisq.network.p2p.peers.PeerManager; -import bisq.network.p2p.peers.keepalive.KeepAliveManager; -import bisq.network.p2p.peers.peerexchange.PeerExchangeManager; -import bisq.network.p2p.storage.messages.BroadcastMessage; - -import bisq.common.ClockWatcher; -import bisq.common.config.Config; -import bisq.common.file.CorruptedStorageFileHandler; -import bisq.common.persistence.PersistenceManager; -import bisq.common.proto.network.NetworkEnvelope; -import bisq.common.proto.network.NetworkProtoResolver; - -import java.time.Clock; - -import java.io.File; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import lombok.extern.slf4j.Slf4j; - -/** - * Contacts a list of hosts and asks them for all the data we do not have. The - * answers are then compiled into buckets of message types. Based on these - * buckets, the Metric reports (for each host) the message types observed and - * their number along with a relative comparison between all hosts. - * - * @author Florian Reimair - * - */ -@Slf4j -public class P2PNetworkLoad extends Metric implements MessageListener, SetupListener { - - private static final String TOR_PROXY_PORT = "run.torProxyPort"; - private static final String MAX_CONNECTIONS = "run.maxConnections"; - private static final String HISTORY_SIZE = "run.historySize"; - private NetworkNode networkNode; - private final File torHiddenServiceDir = new File("metric_" + getName()); - private final ThreadGate hsReady = new ThreadGate(); - private final Map buckets = new ConcurrentHashMap<>(); - - /** - * Buffers the last X message we received. New messages will only be logged in case - * the message isn't already in the history. Note that the oldest message hashes are - * dropped to record newer hashes. - */ - private Map history; - private long lastRun = 0; - - /** - * History implementation using a {@link LinkedHashMap} and its - * {@link LinkedHashMap#removeEldestEntry(Map.Entry)} option. - */ - private static class FixedSizeHistoryTracker extends LinkedHashMap { - final int historySize; - - FixedSizeHistoryTracker(int historySize) { - super(historySize, 10, true); - this.historySize = historySize; - } - - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > historySize; - } - } - - @Override - protected void execute() { - - // in case we do not have a NetworkNode up and running, we create one - if (null == networkNode) { - // prepare the gate - hsReady.engage(); - - // start the network node - networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9053")), - new CoreNetworkProtoResolver(Clock.systemDefaultZone()), false, - new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName()), null); - networkNode.start(this); - - // wait for the HS to be published - hsReady.await(); - - // boot up P2P node - try { - Config config = new Config(); - CorruptedStorageFileHandler corruptedStorageFileHandler = new CorruptedStorageFileHandler(); - int maxConnections = Integer.parseInt(configuration.getProperty(MAX_CONNECTIONS, "12")); - NetworkProtoResolver networkProtoResolver = new CoreNetworkProtoResolver(Clock.systemDefaultZone()); - CorePersistenceProtoResolver persistenceProtoResolver = new CorePersistenceProtoResolver(null, - networkProtoResolver); - DefaultSeedNodeRepository seedNodeRepository = new DefaultSeedNodeRepository(config); - PeerManager peerManager = new PeerManager(networkNode, seedNodeRepository, new ClockWatcher(), - new PersistenceManager<>(torHiddenServiceDir, persistenceProtoResolver, corruptedStorageFileHandler), maxConnections); - - // init file storage - peerManager.readPersisted(() -> { - }); - - PeerExchangeManager peerExchangeManager = new PeerExchangeManager(networkNode, seedNodeRepository, - peerManager); - // updates the peer list every now and then as well - peerExchangeManager - .requestReportedPeersFromSeedNodes(seedNodeRepository.getSeedNodeAddresses().iterator().next()); - - KeepAliveManager keepAliveManager = new KeepAliveManager(networkNode, peerManager); - keepAliveManager.start(); - - networkNode.addMessageListener(this); - } catch (Throwable e) { - e.printStackTrace(); - } - } - - // report - Map report = new HashMap<>(); - - if (lastRun != 0 && System.currentTimeMillis() - lastRun != 0) { - // - normalize to data/minute - double perMinuteFactor = 60000.0 / (System.currentTimeMillis() - lastRun); - - - // - get snapshot so we do not loose data - Set keys = new HashSet<>(buckets.keySet()); - - // - transfer values to report - keys.forEach(key -> { - int value = buckets.get(key).getAndReset(); - if (value != 0) { - report.put(key, String.format("%.2f", value * perMinuteFactor)); - } - }); - - // - report - reporter.report(report, getName()); - } - - // - reset last run - lastRun = System.currentTimeMillis(); - } - - public P2PNetworkLoad(Reporter reporter) { - super(reporter); - } - - @Override - public void configure(Properties properties) { - super.configure(properties); - - history = Collections.synchronizedMap(new FixedSizeHistoryTracker<>(Integer.parseInt(configuration.getProperty(HISTORY_SIZE, "200")))); - } - - /** - * Efficient way to count message occurrences. - */ - private static class Counter { - private int value = 1; - - synchronized int getAndReset() { - try { - return value; - } finally { - value = 0; - } - } - - synchronized void increment() { - value++; - } - } - - @Override - public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { - if (networkEnvelope instanceof BroadcastMessage) { - try { - if (history.get(networkEnvelope.hashCode()) == null) { - history.put(networkEnvelope.hashCode(), null); - buckets.get(networkEnvelope.getClass().getSimpleName()).increment(); - } - } catch (NullPointerException e) { - // use exception handling because we hardly ever need to add a fresh bucket - buckets.put(networkEnvelope.getClass().getSimpleName(), new Counter()); - } - } - } - - @Override - public void onTorNodeReady() { - } - - @Override - public void onHiddenServicePublished() { - // open the gate - hsReady.proceed(); - } - - @Override - public void onSetupFailed(Throwable throwable) { - } - - @Override - public void onRequestCustomBridges() { - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java deleted file mode 100644 index bef8ad22f5c..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * This file is part of Bisq. - * - * bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.OnionParser; -import bisq.monitor.Reporter; -import bisq.monitor.StatisticsHelper; - -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.network.CloseConnectionReason; -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.peers.keepalive.messages.Ping; -import bisq.network.p2p.peers.keepalive.messages.Pong; - -import bisq.common.proto.network.NetworkEnvelope; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import static com.google.common.base.Preconditions.checkNotNull; - -public class P2PRoundTripTime extends P2PSeedNodeSnapshotBase { - - private static final String SAMPLE_SIZE = "run.sampleSize"; - private final Map sentAt = new HashMap<>(); - private Map measurements = new HashMap<>(); - - public P2PRoundTripTime(Reporter reporter) { - super(reporter); - } - - /** - * Use a counter to do statistics. - */ - private class Statistics { - - private final List samples = new ArrayList<>(); - - public synchronized void log(Object message) { - Pong pong = (Pong) message; - Long start = sentAt.get(pong.getRequestNonce()); - if (start != null) - samples.add(System.currentTimeMillis() - start); - } - - public List values() { - return samples; - } - } - - @Override - protected List getRequests() { - List result = new ArrayList<>(); - - Random random = new Random(); - for (int i = 0; i < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1")); i++) - result.add(new Ping(random.nextInt(), 42)); - - return result; - } - - @Override - protected void aboutToSend(NetworkEnvelope message) { - sentAt.put(((Ping) message).getNonce(), System.currentTimeMillis()); - } - - @Override - protected boolean treatMessage(NetworkEnvelope networkEnvelope, Connection connection) { - if (networkEnvelope instanceof Pong) { - checkNotNull(connection.getPeersNodeAddressProperty(), - "although the property is nullable, we need it to not be null"); - - measurements.putIfAbsent(connection.getPeersNodeAddressProperty().getValue(), new Statistics()); - - measurements.get(connection.getPeersNodeAddressProperty().getValue()).log(networkEnvelope); - - connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN); - return true; - } - return false; - } - - @Override - void report() { - // report - measurements.forEach(((nodeAddress, samples) -> - reporter.report(StatisticsHelper.process(samples.values()), - getName() + "." + OnionParser.prettyPrint(nodeAddress)) - )); - // clean up for next round - measurements = new HashMap<>(); - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java deleted file mode 100644 index 3072abea393..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * This file is part of Bisq. - * - * bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.OnionParser; -import bisq.monitor.Reporter; - -import bisq.core.dao.monitoring.model.StateHash; -import bisq.core.dao.monitoring.network.messages.GetBlindVoteStateHashesRequest; -import bisq.core.dao.monitoring.network.messages.GetDaoStateHashesRequest; -import bisq.core.dao.monitoring.network.messages.GetProposalStateHashesRequest; -import bisq.core.dao.monitoring.network.messages.GetStateHashesResponse; - -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; -import bisq.network.p2p.storage.payload.ProtectedStorageEntry; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import bisq.common.proto.network.NetworkEnvelope; - -import java.net.MalformedURLException; - -import java.nio.ByteBuffer; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Contacts a list of hosts and asks them for all the data excluding persisted messages. The - * answers are then compiled into buckets of message types. Based on these - * buckets, the Metric reports (for each host) the message types observed and - * their number. - * - * Furthermore, since the DAO is a thing now, the consistency of the DAO state held by each host is assessed and reported. - * - * @author Florian Reimair - * - */ -@Slf4j -public class P2PSeedNodeSnapshot extends P2PSeedNodeSnapshotBase { - - final Map>> bucketsPerHost = new ConcurrentHashMap<>(); - private int daostateheight = 594000; - private int proposalheight = daostateheight; - private int blindvoteheight = daostateheight; - - /** - * Use a counter to do statistics. - */ - private static class MyStatistics extends Statistics> { - - @Override - public synchronized void log(Object message) { - - // For logging different data types - String className = message.getClass().getSimpleName(); - - buckets.putIfAbsent(className, new HashSet<>()); - buckets.get(className).add(message.hashCode()); - } - } - - public P2PSeedNodeSnapshot(Reporter reporter) { - super(reporter); - } - - protected List getRequests() { - List result = new ArrayList<>(); - - Random random = new Random(); - result.add(new PreliminaryGetDataRequest(random.nextInt(), hashes)); - - result.add(new GetDaoStateHashesRequest(daostateheight, random.nextInt())); - - result.add(new GetProposalStateHashesRequest(proposalheight, random.nextInt())); - - result.add(new GetBlindVoteStateHashesRequest(blindvoteheight, random.nextInt())); - - return result; - } - - /** - * Report all the stuff. Uses the configured reporter directly. - */ - void report() { - - // report - Map report = new HashMap<>(); - // - assemble histograms - bucketsPerHost.forEach((host, statistics) -> statistics.values().forEach((type, set) -> report - .put(OnionParser.prettyPrint(host) + ".numberOfMessages." + type, Integer.toString(set.size())))); - - // - assemble diffs - // - transfer values - Map>> messagesPerHost = new HashMap<>(); - bucketsPerHost.forEach((host, value) -> messagesPerHost.put(OnionParser.prettyPrint(host), value)); - - // - pick reference seed node and its values - String referenceHost = "overall_number_of_unique_messages"; - Map> referenceValues = new HashMap<>(); - messagesPerHost.forEach((host, statistics) -> statistics.values().forEach((type, set) -> { - referenceValues.putIfAbsent(type, new HashSet<>()); - referenceValues.get(type).addAll(set); - })); - - // - calculate diffs - messagesPerHost.forEach( - (host, statistics) -> { - statistics.values().forEach((messageType, set) -> { - try { - report.put(OnionParser.prettyPrint(host) + ".relativeNumberOfMessages." + messageType, - String.valueOf(set.size() - referenceValues.get(messageType).size())); - } catch (MalformedURLException | NullPointerException e) { - log.error("we should never have gotten here", e); - } - }); - try { - report.put(OnionParser.prettyPrint(host) + ".referenceHost", referenceHost); - } catch (MalformedURLException ignore) { - log.error("we should never got here"); - } - }); - - // cleanup for next run - bucketsPerHost.forEach((host, statistics) -> statistics.reset()); - - // when our hash cache exceeds a hard limit, we clear the cache and start anew - if (hashes.size() > 150000) - hashes.clear(); - - // - report - reporter.report(report, getName()); - - // - assemble dao report - Map daoreport = new HashMap<>(); - - // - transcode - Map> perType = new HashMap<>(); - daoData.forEach((nodeAddress, daostatistics) -> daostatistics.values().forEach((type, tuple) -> { - perType.putIfAbsent(type, new HashMap<>()); - perType.get(type).put(nodeAddress, tuple); - })); - - // - process dao data - perType.forEach((type, nodeAddressTupleMap) -> { - // - find head - int head = nodeAddressTupleMap.values().stream().max(Comparator.comparingLong(Tuple::getHeight)) - .map(value -> (int) value.height) - .orElse(0); - int oldest = nodeAddressTupleMap.values().stream().min(Comparator.comparingLong(Tuple::getHeight)) - .map(value -> (int) value.height) - .orElse(0); - - // - update queried height - if (type.contains("DaoState")) - daostateheight = oldest - 20; - else if (type.contains("Proposal")) - proposalheight = oldest - 20; - else - blindvoteheight = oldest - 20; - - // - calculate diffs - nodeAddressTupleMap.forEach((nodeAddress, tuple) -> daoreport.put(type + "." + OnionParser.prettyPrint(nodeAddress) + ".head", Long.toString(tuple.height - head))); - - // - memorize hashes - Map hitcount = new HashMap<>(); - nodeAddressTupleMap.forEach((nodeAddress, tuple) -> { - ByteBuffer hash = ByteBuffer.wrap(tuple.hash); - if (hitcount.containsKey(hash)) { - hitcount.put(hash, hitcount.get(hash) + 1); - } else - hitcount.put(hash, 1); - }); - - hitcount.clear(); - - nodeAddressTupleMap.forEach((nodeAddress, tuple) -> - daoreport.put(type + "." + OnionParser.prettyPrint(nodeAddress) + ".hash", - Integer.toString(Arrays.asList(hitcount.entrySet().stream() - .sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue())) - .map(Map.Entry::getKey).toArray()).indexOf(ByteBuffer - .wrap(tuple.hash))))); - - // - report reference head - daoreport.put(type + ".referenceHead", Integer.toString(head)); - }); - - daoData.clear(); - - // - report - reporter.report(daoreport, "DaoStateSnapshot"); - } - - private static class Tuple { - @Getter - private final long height; - private final byte[] hash; - - Tuple(long height, byte[] hash) { - this.height = height; - this.hash = hash; - } - } - - private class DaoStatistics extends Statistics { - - @Override - public void log(Object message) { - // get last entry - StateHash last = (StateHash) ((GetStateHashesResponse) message).getStateHashes().get(((GetStateHashesResponse) message).getStateHashes().size() - 1); - - // For logging different data types - String className = last.getClass().getSimpleName(); - - buckets.putIfAbsent(className, new Tuple(last.getHeight(), last.getHash())); - } - } - - private final Map> daoData = new ConcurrentHashMap<>(); - - protected boolean treatMessage(NetworkEnvelope networkEnvelope, Connection connection) { - checkNotNull(connection.getPeersNodeAddressProperty(), - "although the property is nullable, we need it to not be null"); - - if (networkEnvelope instanceof GetDataResponse) { - - Statistics result = new MyStatistics(); - - GetDataResponse dataResponse = (GetDataResponse) networkEnvelope; - final Set dataSet = dataResponse.getDataSet(); - dataSet.forEach(e -> { - final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); - if (protectedStoragePayload == null) { - log.warn("StoragePayload was null: {}", networkEnvelope.toString()); - return; - } - - result.log(protectedStoragePayload); - }); - - dataResponse.getPersistableNetworkPayloadSet().forEach(persistableNetworkPayload -> { - // memorize message hashes - //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; - //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); - - //hashes.add(bytes); - - hashes.add(persistableNetworkPayload.getHash()); - }); - - bucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), result); - return true; - } else if (networkEnvelope instanceof GetStateHashesResponse) { - daoData.putIfAbsent(connection.getPeersNodeAddressProperty().getValue(), new DaoStatistics()); - - daoData.get(connection.getPeersNodeAddressProperty().getValue()).log(networkEnvelope); - - return true; - } - return false; - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshotBase.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshotBase.java deleted file mode 100644 index 9550b3dae9f..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshotBase.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * This file is part of Bisq. - * - * bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.AvailableTor; -import bisq.monitor.Metric; -import bisq.monitor.Monitor; -import bisq.monitor.OnionParser; -import bisq.monitor.Reporter; -import bisq.monitor.ThreadGate; - -import bisq.core.account.witness.AccountAgeWitnessStore; -import bisq.core.proto.network.CoreNetworkProtoResolver; -import bisq.core.proto.persistable.CorePersistenceProtoResolver; -import bisq.core.trade.statistics.TradeStatistics3Store; - -import bisq.network.p2p.CloseConnectionMessage; -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.network.MessageListener; -import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.network.TorNetworkNode; - -import bisq.common.app.Version; -import bisq.common.config.BaseCurrencyNetwork; -import bisq.common.persistence.PersistenceManager; -import bisq.common.proto.network.NetworkEnvelope; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.SettableFuture; - -import java.time.Clock; - -import java.io.File; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -import org.jetbrains.annotations.NotNull; - -/** - * Contacts a list of hosts and asks them for all the data excluding persisted messages. The - * answers are then compiled into buckets of message types. Based on these - * buckets, the Metric reports (for each host) the message types observed and - * their number. - * - * @author Florian Reimair - * - */ -@Slf4j -public abstract class P2PSeedNodeSnapshotBase extends Metric implements MessageListener { - - private static final String HOSTS = "run.hosts"; - private static final String TOR_PROXY_PORT = "run.torProxyPort"; - private static final String DATABASE_DIR = "run.dbDir"; - final Map> bucketsPerHost = new ConcurrentHashMap<>(); - private final ThreadGate gate = new ThreadGate(); - protected final Set hashes = new TreeSet<>(Arrays::compare); - - /** - * Statistics Interface for use with derived classes. - * - * @param the value type of the statistics implementation - */ - protected abstract static class Statistics { - protected final Map buckets = new HashMap<>(); - - abstract void log(Object message); - - Map values() { - return buckets; - } - - void reset() { - buckets.clear(); - } - } - - public P2PSeedNodeSnapshotBase(Reporter reporter) { - super(reporter); - } - - @Override - public void configure(Properties properties) { - super.configure(properties); - - if (hashes.isEmpty() && configuration.getProperty(DATABASE_DIR) != null) { - File dir = new File(configuration.getProperty(DATABASE_DIR)); - String networkPostfix = "_" + BaseCurrencyNetwork.values()[Version.getBaseCurrencyNetwork()].toString(); - try { - CorePersistenceProtoResolver persistenceProtoResolver = new CorePersistenceProtoResolver(null, null); - - //TODO will not work with historical data... should be refactored to re-use code for reading resource files - TradeStatistics3Store tradeStatistics3Store = new TradeStatistics3Store(); - PersistenceManager tradeStatistics3PersistenceManager = new PersistenceManager<>(dir, - persistenceProtoResolver, null); - tradeStatistics3PersistenceManager.initialize(tradeStatistics3Store, - tradeStatistics3Store.getDefaultStorageFileName() + networkPostfix, - PersistenceManager.Source.NETWORK); - TradeStatistics3Store persistedTradeStatistics3Store = tradeStatistics3PersistenceManager.getPersisted(); - if (persistedTradeStatistics3Store != null) { - tradeStatistics3Store.getMap().putAll(persistedTradeStatistics3Store.getMap()); - } - hashes.addAll(tradeStatistics3Store.getMap().keySet().stream() - .map(byteArray -> byteArray.bytes).collect(Collectors.toSet())); - - AccountAgeWitnessStore accountAgeWitnessStore = new AccountAgeWitnessStore(); - PersistenceManager accountAgeWitnessPersistenceManager = new PersistenceManager<>(dir, - persistenceProtoResolver, null); - accountAgeWitnessPersistenceManager.initialize(accountAgeWitnessStore, - accountAgeWitnessStore.getDefaultStorageFileName() + networkPostfix, - PersistenceManager.Source.NETWORK); - AccountAgeWitnessStore persistedAccountAgeWitnessStore = accountAgeWitnessPersistenceManager.getPersisted(); - if (persistedAccountAgeWitnessStore != null) { - accountAgeWitnessStore.getMap().putAll(persistedAccountAgeWitnessStore.getMap()); - } - hashes.addAll(accountAgeWitnessStore.getMap().keySet().stream() - .map(byteArray -> byteArray.bytes).collect(Collectors.toSet())); - } catch (NullPointerException e) { - // in case there is no store file - log.error("There is no storage file where there should be one: {}", dir.getAbsolutePath()); - } - } - } - - @Override - protected void execute() { - // start the network node - final NetworkNode networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9054")), - new CoreNetworkProtoResolver(Clock.systemDefaultZone()), false, - new AvailableTor(Monitor.TOR_WORKING_DIR, "unused"), null); - // we do not need to start the networkNode, as we do not need the HS - //networkNode.start(this); - - // clear our buckets - bucketsPerHost.clear(); - - getRequests().forEach(getDataRequest -> send(networkNode, getDataRequest)); - - report(); - } - - protected abstract List getRequests(); - - protected void send(NetworkNode networkNode, NetworkEnvelope message) { - - ArrayList threadList = new ArrayList<>(); - - // for each configured host - for (String current : configuration.getProperty(HOSTS, "").split(",")) { - threadList.add(new Thread(() -> { - - try { - // parse Url - NodeAddress target = OnionParser.getNodeAddress(current); - - // do the data request - aboutToSend(message); - SettableFuture future = networkNode.sendMessage(target, message); - - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(Connection connection) { - connection.addMessageListener(P2PSeedNodeSnapshotBase.this); - } - - @Override - public void onFailure(@NotNull Throwable throwable) { - gate.proceed(); - log.error( - "Sending {} failed. That is expected if the peer is offline.\n\tException={}", message.getClass().getSimpleName(), throwable.getMessage()); - } - }, MoreExecutors.directExecutor()); - - } catch (Exception e) { - gate.proceed(); // release the gate on error - e.printStackTrace(); - } - }, current)); - } - - gate.engage(threadList.size()); - - // start all threads and wait until they all finished. We do that so we can - // minimize the time between querying the hosts and therefore the chance of - // inconsistencies. - threadList.forEach(Thread::start); - - gate.await(); - } - - protected void aboutToSend(NetworkEnvelope message) { - } - - /** - * Report all the stuff. Uses the configured reporter directly. - */ - abstract void report(); - - @Override - public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { - if (treatMessage(networkEnvelope, connection)) { - gate.proceed(); - } else if (networkEnvelope instanceof CloseConnectionMessage) { - gate.unlock(); - } else { - log.warn("Got an unexpected message of type <{}>", - networkEnvelope.getClass().getSimpleName()); - } - connection.removeMessageListener(this); - } - - protected abstract boolean treatMessage(NetworkEnvelope networkEnvelope, Connection connection); -} diff --git a/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java b/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java deleted file mode 100644 index fd10691f727..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * This file is part of Bisq. - * - * bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.Metric; -import bisq.monitor.OnionParser; -import bisq.monitor.Reporter; - -import bisq.asset.Asset; -import bisq.asset.AssetRegistry; - -import bisq.network.p2p.NodeAddress; - -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import com.runjva.sourceforge.jsocks.protocol.Socks5Proxy; -import com.runjva.sourceforge.jsocks.protocol.SocksSocket; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import lombok.extern.slf4j.Slf4j; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Fetches fee and price data from the configured price nodes. - * Based on the work of HarryMcFinned. - * - * @author Florian Reimair - * @author HarryMcFinned - * - */ -@Slf4j -public class PriceNodeStats extends Metric { - - private static final String HOSTS = "run.hosts"; - private static final String IGNORE = "dashTxFee ltcTxFee dogeTxFee"; - // poor mans JSON parser - private final Pattern stringNumberPattern = Pattern.compile("\"(.+)\" ?: ?(\\d+)"); - private final Pattern pricePattern = Pattern.compile("\"price\" ?: ?([\\d.]+)"); - private final Pattern currencyCodePattern = Pattern.compile("\"currencyCode\" ?: ?\"([A-Z]+)\""); - private final List assets = Arrays.asList(new AssetRegistry().stream().map(Asset::getTickerSymbol).toArray()); - - public PriceNodeStats(Reporter reporter) { - super(reporter); - } - - @Override - protected void execute() { - try { - // fetch proxy - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - Socks5Proxy proxy = tor.getProxy(); - - String[] hosts = configuration.getProperty(HOSTS, "").split(","); - - Collections.shuffle(Arrays.asList(hosts)); - - // for each configured host - for (String current : hosts) { - Map result = new HashMap<>(); - // parse Url - NodeAddress tmp = OnionParser.getNodeAddress(current); - - // connect - try { - SocksSocket socket = new SocksSocket(proxy, tmp.getHostName(), tmp.getPort()); - - // prepare to receive data - BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream())); - - // ask for fee data - PrintWriter out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(socket.getOutputStream()))); - out.println("GET /getFees/"); - out.println(); - out.flush(); - - // sift through the received lines and see if we got something json-like - String line; - while ((line = in.readLine()) != null) { - Matcher matcher = stringNumberPattern.matcher(line); - if (matcher.find()) - if (!IGNORE.contains(matcher.group(1))) - result.put("fees." + matcher.group(1), matcher.group(2)); - } - - in.close(); - out.close(); - socket.close(); - - // connect - socket = new SocksSocket(proxy, tmp.getHostName(), tmp.getPort()); - - // prepare to receive data - in = new BufferedReader(new InputStreamReader(socket.getInputStream())); - - // ask for exchange rate data - out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(socket.getOutputStream()))); - out.println("GET /getAllMarketPrices/"); - out.println(); - out.flush(); - - String currencyCode = ""; - while ((line = in.readLine()) != null) { - Matcher currencyCodeMatcher = currencyCodePattern.matcher(line); - Matcher priceMatcher = pricePattern.matcher(line); - if (currencyCodeMatcher.find()) { - currencyCode = currencyCodeMatcher.group(1); - if (!assets.contains(currencyCode)) - currencyCode = ""; - } else if (!"".equals(currencyCode) && priceMatcher.find()) - result.put("price." + currencyCode, priceMatcher.group(1)); - } - - // close all the things - in.close(); - out.close(); - socket.close(); - - // report - reporter.report(result, getName()); - - // only ask for data as long as we got none - if (!result.isEmpty()) - break; - } catch (IOException e) { - log.error("{} seems to be down. Trying next configured price node.", tmp.getHostName()); - e.printStackTrace(); - } - } - } catch (TorCtlException | IOException e) { - e.printStackTrace(); - } - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java b/monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java deleted file mode 100644 index 21be8526eff..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.Metric; -import bisq.monitor.Monitor; -import bisq.monitor.Reporter; -import bisq.monitor.ThreadGate; - -import org.berndpruenster.netlayer.tor.HiddenServiceSocket; - -import java.io.File; - -import lombok.extern.slf4j.Slf4j; - -/** - * A Metric to measure the startup time of a Tor Hidden Service on a already - * running Tor. - * - * @author Florian Reimair - */ -@Slf4j -public class TorHiddenServiceStartupTime extends Metric { - - private static final String SERVICE_PORT = "run.servicePort"; - private static final String LOCAL_PORT = "run.localPort"; - private final String hiddenServiceDirectory = "metric_" + getName(); - private final ThreadGate gate = new ThreadGate(); - - public TorHiddenServiceStartupTime(Reporter reporter) { - super(reporter); - } - - @Override - protected void execute() { - // prepare settings. Fetch them every time we run the Metric so we do not have to - // restart on a config update - int localPort = Integer.parseInt(configuration.getProperty(LOCAL_PORT, "9998")); - int servicePort = Integer.parseInt(configuration.getProperty(SERVICE_PORT, "9999")); - - // clear directory so we get a new onion address every time - new File(Monitor.TOR_WORKING_DIR + "/" + hiddenServiceDirectory).delete(); - - log.debug("creating the hidden service"); - - gate.engage(); - - // start timer - we do not need System.nanoTime as we expect our result to be in - // the range of tenth of seconds. - long start = System.currentTimeMillis(); - - HiddenServiceSocket hiddenServiceSocket = new HiddenServiceSocket(localPort, hiddenServiceDirectory, - servicePort); - hiddenServiceSocket.addReadyListener(socket -> { - // stop the timer and report - reporter.report(System.currentTimeMillis() - start, getName()); - log.debug("the hidden service is ready"); - gate.proceed(); - return null; - }); - - gate.await(); - log.debug("going to revoke the hidden service..."); - hiddenServiceSocket.close(); - log.debug("[going to revoke the hidden service...] done"); - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java deleted file mode 100644 index a40c0261662..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.Metric; -import bisq.monitor.OnionParser; -import bisq.monitor.Reporter; -import bisq.monitor.StatisticsHelper; - -import bisq.network.p2p.NodeAddress; - -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import com.runjva.sourceforge.jsocks.protocol.Socks5Proxy; -import com.runjva.sourceforge.jsocks.protocol.SocksSocket; - -import java.io.IOException; - -import java.util.ArrayList; -import java.util.List; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * A Metric to measure the round-trip time to the Bisq seed nodes via plain tor. - * - * @author Florian Reimair - */ -public class TorRoundTripTime extends Metric { - - private static final String SAMPLE_SIZE = "run.sampleSize"; - private static final String HOSTS = "run.hosts"; - - public TorRoundTripTime(Reporter reporter) { - super(reporter); - } - - @Override - protected void execute() { - SocksSocket socket; - try { - // fetch proxy - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - Socks5Proxy proxy = tor.getProxy(); - - // for each configured host - for (String current : configuration.getProperty(HOSTS, "").split(",")) { - // parse Url - NodeAddress tmp = OnionParser.getNodeAddress(current); - - List samples = new ArrayList<>(); - - while (samples.size() < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1"))) { - // start timer - we do not need System.nanoTime as we expect our result to be in - // seconds time. - long start = System.currentTimeMillis(); - - // connect - socket = new SocksSocket(proxy, tmp.getHostName(), tmp.getPort()); - - // by the time we get here, we are connected - samples.add(System.currentTimeMillis() - start); - - // cleanup - socket.close(); - } - - // report - reporter.report(StatisticsHelper.process(samples), getName()); - } - } catch (TorCtlException | IOException e) { - e.printStackTrace(); - } - } -} diff --git a/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java b/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java deleted file mode 100644 index 9ea3198e197..00000000000 --- a/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor.metric; - -import bisq.monitor.Metric; -import bisq.monitor.Reporter; - -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; -import org.berndpruenster.netlayer.tor.Torrc; - -import java.io.File; -import java.io.IOException; - -import java.util.LinkedHashMap; -import java.util.Properties; - -/** - * A Metric to measure the deployment and startup time of the packaged Tor - * binaries. - * - * @author Florian Reimair - */ -public class TorStartupTime extends Metric { - - private static final String SOCKS_PORT = "run.socksPort"; - private final File torWorkingDirectory = new File("monitor/work/metric_torStartupTime"); - private Torrc torOverrides; - - public TorStartupTime(Reporter reporter) { - super(reporter); - } - - @Override - public void configure(Properties properties) { - super.configure(properties); - - synchronized (this) { - LinkedHashMap overrides = new LinkedHashMap<>(); - overrides.put("SOCKSPort", configuration.getProperty(SOCKS_PORT, "90500")); - - try { - torOverrides = new Torrc(overrides); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - @Override - protected void execute() { - // cleanup installation - torWorkingDirectory.delete(); - Tor tor = null; - // start timer - we do not need System.nanoTime as we expect our result to be in - // tenth of seconds time. - long start = System.currentTimeMillis(); - - try { - tor = new NativeTor(torWorkingDirectory, null, torOverrides); - - // stop the timer and set its timestamp - reporter.report(System.currentTimeMillis() - start, getName()); - } catch (TorCtlException e) { - e.printStackTrace(); - } finally { - // cleanup - if (tor != null) - tor.shutdown(); - } - } -} diff --git a/monitor/src/main/java/bisq/monitor/reporter/ConsoleReporter.java b/monitor/src/main/java/bisq/monitor/reporter/ConsoleReporter.java deleted file mode 100644 index f34dc2d5679..00000000000 --- a/monitor/src/main/java/bisq/monitor/reporter/ConsoleReporter.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor.reporter; - -import bisq.monitor.Reporter; - -import bisq.common.app.Version; -import bisq.common.config.BaseCurrencyNetwork; - -import java.util.HashMap; -import java.util.Map; - -/** - * A simple console reporter. - * - * @author Florian Reimair - */ -public class ConsoleReporter extends Reporter { - - @Override - public void report(long value, String prefix) { - HashMap result = new HashMap<>(); - result.put("", String.valueOf(value)); - report(result, prefix); - - } - - @Override - public void report(long value) { - HashMap result = new HashMap<>(); - result.put("", String.valueOf(value)); - report(result); - } - - @Override - public void report(Map values, String prefix) { - String timestamp = String.valueOf(System.currentTimeMillis()); - values.forEach((key, value) -> { - report(key, value, timestamp, prefix); - }); - } - - @Override - public void report(String key, String value, String timestamp, String prefix) { - System.err.println("Report: bisq" + (Version.getBaseCurrencyNetwork() != 0 ? "-" + BaseCurrencyNetwork.values()[Version.getBaseCurrencyNetwork()].getNetwork() : "") - + (prefix.isEmpty() ? "" : "." + prefix) - + (key.isEmpty() ? "" : "." + key) - + " " + value + " " + timestamp); - } - - @Override - public void report(Map values) { - report(values, ""); - } -} diff --git a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java b/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java deleted file mode 100644 index dc0334ff40c..00000000000 --- a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor.reporter; - -import bisq.monitor.OnionParser; -import bisq.monitor.Reporter; - -import bisq.network.p2p.NodeAddress; - -import bisq.common.app.Version; -import bisq.common.config.BaseCurrencyNetwork; - -import org.berndpruenster.netlayer.tor.TorSocket; - -import com.google.common.base.Charsets; - -import java.net.Socket; - -import java.io.IOException; - -import java.util.HashMap; -import java.util.Map; - -/** - * Reports our findings to a graphite service. - * - * @author Florian Reimair - */ -public class GraphiteReporter extends Reporter { - - @Override - public void report(long value, String prefix) { - HashMap result = new HashMap<>(); - result.put("", String.valueOf(value)); - report(result, prefix); - - } - - @Override - public void report(long value) { - report(value, ""); - } - - @Override - public void report(Map values, String prefix) { - String timestamp = String.valueOf(System.currentTimeMillis()); - values.forEach((key, value) -> { - - report(key, value, timestamp, prefix); - try { - // give Tor some slack - // TODO maybe use the pickle protocol? - // https://graphite.readthedocs.io/en/latest/feeding-carbon.html - Thread.sleep(100); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - }); - } - - @Override - public void report(String key, String value, String timeInMilliseconds, String prefix) { - // https://graphite.readthedocs.io/en/latest/feeding-carbon.html - String report = "bisq" + (Version.getBaseCurrencyNetwork() != 0 ? "-" + BaseCurrencyNetwork.values()[Version.getBaseCurrencyNetwork()].getNetwork() : "") - + (prefix.isEmpty() ? "" : "." + prefix) - + (key.isEmpty() ? "" : "." + key) - + " " + value + " " + Long.parseLong(timeInMilliseconds) / 1000 + "\n"; - - try { - NodeAddress nodeAddress = OnionParser.getNodeAddress(configuration.getProperty("serviceUrl")); - Socket socket; - if (nodeAddress.getFullAddress().contains(".onion")) - socket = new TorSocket(nodeAddress.getHostName(), nodeAddress.getPort()); - else - socket = new Socket(nodeAddress.getHostName(), nodeAddress.getPort()); - - socket.getOutputStream().write(report.getBytes(Charsets.UTF_8)); - socket.close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } - - @Override - public void report(Map values) { - report(values, ""); - } -} diff --git a/monitor/src/main/resources/logback.xml b/monitor/src/main/resources/logback.xml deleted file mode 100644 index 28831d6fc3b..00000000000 --- a/monitor/src/main/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - diff --git a/monitor/src/main/resources/metrics.properties b/monitor/src/main/resources/metrics.properties deleted file mode 100644 index 83a66920ca6..00000000000 --- a/monitor/src/main/resources/metrics.properties +++ /dev/null @@ -1,79 +0,0 @@ -## System configuration - -# true overwrites the reporters picked by the developers (for debugging for example) (defaults to false) -System.useConsoleReporter=true - -# 0 -> BTC_MAINNET, 1 -> BTC_TESTNET (default) -System.baseCurrencyNetwork=0 - -## Each Metric is configured via a set of properties. -## -## The minimal set of properties required to run a Metric is: -## -## YourMetricName.enabled=true|false -## YourMetricName.run.interval=10 [seconds] - -#Edit and uncomment the lines below to your liking - -#TorStartupTime Metric -TorStartupTime.enabled=false -TorStartupTime.run.interval=100 -TorStartupTime.run.socksPort=90500 - -TorRoundTripTime.enabled=false -TorRoundTripTime.run.interval=100 -TorRoundTripTime.run.sampleSize=3 -# torproject.org hidden service -TorRoundTripTime.run.hosts=http://expyuzz4wqqyqhjn.onion:80 - -#TorHiddenServiceStartupTime Metric -TorHiddenServiceStartupTime.enabled=false -TorHiddenServiceStartupTime.run.interval=100 -TorHiddenServiceStartupTime.run.localPort=90501 -TorHiddenServiceStartupTime.run.servicePort=90511 - -#P2PRoundTripTime Metric -P2PRoundTripTime.enabled=false -P2PRoundTripTime.run.interval=100 -P2PRoundTripTime.run.sampleSize=5 -P2PRoundTripTime.run.hosts=723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 -P2PRoundTripTime.run.torProxyPort=9060 - -#P2PNetworkLoad Metric -P2PNetworkLoad.enabled=false -P2PNetworkLoad.run.interval=100 -P2PNetworkLoad.run.torProxyPort=9061 -P2PNetworkLoad.run.historySize=200 - -#P2PSeedNodeSnapshotBase Metric -P2PSeedNodeSnapshot.enabled=true -P2PSeedNodeSnapshot.run.dbDir=bisq/p2p/build/resources/main/ -P2PSeedNodeSnapshot.run.interval=24 -P2PSeedNodeSnapshot.run.hosts=3f3cu2yw7u457ztq.onion:8000, 723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 -P2PSeedNodeSnapshot.run.torProxyPort=9062 - -#P2PMarketStats Metric -P2PMarketStats.enabled=false -P2PMarketStats.run.interval=37 -P2PMarketStats.run.dbDir=bisq/p2p/build/resources/main/ -P2PMarketStats.run.hosts=ef5qnzx6znifo3df.onion:8000 -P2PMarketStats.run.torProxyPort=9063 - -#PriceNodeStats Metric -PriceNodeStats.enabled=false -PriceNodeStats.run.interval=42 -PriceNodeStats.run.hosts=http://xc3nh4juf2hshy7e.onion, http://44mgyoe2b6oqiytt.onion, http://62nvujg5iou3vu3i.onion, http://ceaanhbvluug4we6.onion, http://gztmprecgqjq64zh.onion/ - -#MarketStats Metric -MarketStats.enabled=false -MarketStats.run.interval=191 - -#Another Metric -Another.run.interval=5 - -## Reporters are configured via a set of properties as well. -## -## In contrast to Metrics, Reporters do not have a minimal set of properties. - -#GraphiteReporter -GraphiteReporter.serviceUrl=k6evlhg44acpchtc.onion:2003 diff --git a/monitor/src/test/java/bisq/monitor/MonitorInfrastructureTests.java b/monitor/src/test/java/bisq/monitor/MonitorInfrastructureTests.java deleted file mode 100644 index 5e1fc248308..00000000000 --- a/monitor/src/test/java/bisq/monitor/MonitorInfrastructureTests.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.reporter.ConsoleReporter; - -import java.util.HashMap; -import java.util.Properties; -import java.util.concurrent.ExecutionException; - -import org.junit.Assert; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -@Disabled -public class MonitorInfrastructureTests { - - /** - * A dummy metric for development purposes. - */ - public class Dummy extends Metric { - - public Dummy() { - super(new ConsoleReporter()); - } - - public boolean active() { - return enabled(); - } - - @Override - protected void execute() { - try { - Thread.sleep(50000); - - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - - @ParameterizedTest - @ValueSource(strings = {"empty", "no interval", "typo"}) - public void basicConfigurationError(String configuration) { - HashMap lut = new HashMap<>(); - lut.put("empty", new Properties()); - Properties noInterval = new Properties(); - noInterval.put("Dummy.enabled", "true"); - lut.put("no interval", noInterval); - Properties typo = new Properties(); - typo.put("Dummy.enabled", "true"); - //noinspection SpellCheckingInspection - typo.put("Dummy.run.inteval", "1"); - lut.put("typo", typo); - - Dummy DUT = new Dummy(); - DUT.configure(lut.get(configuration)); - Assert.assertFalse(DUT.active()); - } - - @Test - public void basicConfigurationSuccess() throws Exception { - Properties correct = new Properties(); - correct.put("Dummy.enabled", "true"); - correct.put("Dummy.run.interval", "1"); - - Dummy DUT = new Dummy(); - DUT.configure(correct); - Assert.assertTrue(DUT.active()); - - // graceful shutdown - Metric.haltAllMetrics(); - } - - @Test - public void reloadConfig() throws InterruptedException, ExecutionException { - // our dummy - Dummy DUT = new Dummy(); - - // a second dummy to run as well - Dummy DUT2 = new Dummy(); - DUT2.setName("Dummy2"); - Properties dummy2Properties = new Properties(); - dummy2Properties.put("Dummy2.enabled", "true"); - dummy2Properties.put("Dummy2.run.interval", "1"); - DUT2.configure(dummy2Properties); - - // disable - DUT.configure(new Properties()); - Assert.assertFalse(DUT.active()); - Assert.assertTrue(DUT2.active()); - - // enable - Properties properties = new Properties(); - properties.put("Dummy.enabled", "true"); - properties.put("Dummy.run.interval", "1"); - DUT.configure(properties); - Assert.assertTrue(DUT.active()); - Assert.assertTrue(DUT2.active()); - - // disable again - DUT.configure(new Properties()); - Assert.assertFalse(DUT.active()); - Assert.assertTrue(DUT2.active()); - - // enable again - DUT.configure(properties); - Assert.assertTrue(DUT.active()); - Assert.assertTrue(DUT2.active()); - - // graceful shutdown - Metric.haltAllMetrics(); - } - - @Test - public void shutdown() { - Dummy DUT = new Dummy(); - DUT.setName("Dummy"); - Properties dummyProperties = new Properties(); - dummyProperties.put("Dummy.enabled", "true"); - dummyProperties.put("Dummy.run.interval", "1"); - DUT.configure(dummyProperties); - try { - Thread.sleep(2000); - Metric.haltAllMetrics(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } -} diff --git a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java deleted file mode 100644 index 25a11f976bc..00000000000 --- a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.metric.P2PNetworkLoad; -import bisq.monitor.reporter.ConsoleReporter; - -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import java.util.Map; -import java.util.Properties; - -import org.junit.Assert; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Test the round trip time metric against the hidden service of tor project.org. - * - * @author Florian Reimair - */ -@Disabled -class P2PNetworkLoadTests { - - /** - * A dummy Reporter for development purposes. - */ - private class DummyReporter extends ConsoleReporter { - - private Map results; - - @Override - public void report(long value) { - Assert.fail(); - } - - Map hasResults() { - return results; - } - - @Override - public void report(Map values) { - Assert.fail(); - } - - @Override - public void report(long value, String prefix) { - Assert.fail(); - } - - @Override - public void report(Map values, String prefix) { - super.report(values, prefix); - results = values; - } - } - - @BeforeAll - static void setup() throws TorCtlException { - // simulate the tor instance available to all metrics - Tor.setDefault(new NativeTor(Monitor.TOR_WORKING_DIR)); - } - - @Test - void run() throws Exception { - DummyReporter reporter = new DummyReporter(); - - // configure - Properties configuration = new Properties(); - configuration.put("P2PNetworkLoad.enabled", "true"); - configuration.put("P2PNetworkLoad.run.interval", "10"); - configuration.put("P2PNetworkLoad.run.hosts", - "http://fl3mmribyxgrv63c.onion:8000, http://3f3cu2yw7u457ztq.onion:8000"); - - Metric DUT = new P2PNetworkLoad(reporter); - // start - DUT.configure(configuration); - - // give it some time to start and then stop - while (!DUT.enabled()) - Thread.sleep(500); - Thread.sleep(20000); - - Metric.haltAllMetrics(); - - // observe results - Map results = reporter.hasResults(); - Assert.assertFalse(results.isEmpty()); - } - - @AfterAll - static void cleanup() { - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - tor.shutdown(); - } -} diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java deleted file mode 100644 index a10dde58095..00000000000 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.metric.P2PRoundTripTime; -import bisq.monitor.reporter.ConsoleReporter; - -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import java.util.Map; -import java.util.Properties; - -import org.junit.Assert; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Test the round trip time metric against the hidden service of tor project.org. - * - * @author Florian Reimair - */ -@Disabled -class P2PRoundTripTimeTests { - - /** - * A dummy Reporter for development purposes. - */ - private class DummyReporter extends ConsoleReporter { - - private Map results; - - @Override - public void report(long value) { - Assert.fail(); - } - - Map hasResults() { - return results; - } - - @Override - public void report(Map values) { - Assert.fail(); - } - - @Override - public void report(long value, String prefix) { - Assert.fail(); - } - - @Override - public void report(Map values, String prefix) { - super.report(values, prefix); - results = values; - } - } - - @BeforeAll - static void setup() throws TorCtlException { - // simulate the tor instance available to all metrics - Tor.setDefault(new NativeTor(Monitor.TOR_WORKING_DIR)); - } - - @ParameterizedTest - @ValueSource(strings = {"default", "3", "4", "10"}) - void run(String sampleSize) throws Exception { - DummyReporter reporter = new DummyReporter(); - - // configure - Properties configuration = new Properties(); - configuration.put("P2PRoundTripTime.enabled", "true"); - configuration.put("P2PRoundTripTime.run.interval", "2"); - if (!"default".equals(sampleSize)) - configuration.put("P2PRoundTripTime.run.sampleSize", sampleSize); - // torproject.org hidden service - configuration.put("P2PRoundTripTime.run.hosts", "http://fl3mmribyxgrv63c.onion:8000"); - configuration.put("P2PRoundTripTime.run.torProxyPort", "9052"); - - Metric DUT = new P2PRoundTripTime(reporter); - // start - DUT.configure(configuration); - - // give it some time to start and then stop - while (!DUT.enabled()) - Thread.sleep(2000); - - Metric.haltAllMetrics(); - - // observe results - Map results = reporter.hasResults(); - Assert.assertFalse(results.isEmpty()); - Assert.assertEquals(results.get("sampleSize"), sampleSize.equals("default") ? "1" : sampleSize); - - Integer p25 = Integer.valueOf(results.get("p25")); - Integer p50 = Integer.valueOf(results.get("p50")); - Integer p75 = Integer.valueOf(results.get("p75")); - Integer min = Integer.valueOf(results.get("min")); - Integer max = Integer.valueOf(results.get("max")); - Integer average = Integer.valueOf(results.get("average")); - - Assert.assertTrue(0 < min); - Assert.assertTrue(min <= p25 && p25 <= p50); - Assert.assertTrue(p50 <= p75); - Assert.assertTrue(p75 <= max); - Assert.assertTrue(min <= average && average <= max); - } - - @AfterAll - static void cleanup() { - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - tor.shutdown(); - } -} diff --git a/monitor/src/test/java/bisq/monitor/PriceNodeStatsTests.java b/monitor/src/test/java/bisq/monitor/PriceNodeStatsTests.java deleted file mode 100644 index b424b9878df..00000000000 --- a/monitor/src/test/java/bisq/monitor/PriceNodeStatsTests.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.metric.PriceNodeStats; - -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import java.io.File; - -import java.util.Map; -import java.util.Properties; - -import org.junit.Assert; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * @author Florian Reimair - */ -@Disabled -public class PriceNodeStatsTests { - - private final static File torWorkingDirectory = new File("monitor/" + PriceNodeStatsTests.class.getSimpleName()); - - /** - * A dummy Reporter for development purposes. - */ - private class DummyReporter extends Reporter { - - private Map results; - - @Override - public void report(long value) { - Assert.fail(); - } - - public Map results() { - return results; - } - - @Override - public void report(Map values) { - results = values; - } - - @Override - public void report(Map values, String prefix) { - report(values); - } - - @Override - public void report(String key, String value, String timestamp, String prefix) { - - } - - @Override - public void report(long value, String prefix) { - report(value); - } - } - - @BeforeAll - public static void setup() throws TorCtlException { - // simulate the tor instance available to all metrics - Tor.setDefault(new NativeTor(torWorkingDirectory)); - } - - @Test - public void connect() { - DummyReporter reporter = new DummyReporter(); - Metric DUT = new PriceNodeStats(reporter); - - - Properties configuration = new Properties(); - configuration.put("PriceNodeStats.run.hosts", "http://5bmpx76qllutpcyp.onion"); - - DUT.configure(configuration); - - DUT.execute(); - - Assert.assertNotNull(reporter.results()); - Assert.assertTrue(reporter.results.size() > 0); - } - - @AfterAll - public static void cleanup() { - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - tor.shutdown(); - torWorkingDirectory.delete(); - } - -} diff --git a/monitor/src/test/java/bisq/monitor/TorHiddenServiceStartupTimeTests.java b/monitor/src/test/java/bisq/monitor/TorHiddenServiceStartupTimeTests.java deleted file mode 100644 index d517bcd5659..00000000000 --- a/monitor/src/test/java/bisq/monitor/TorHiddenServiceStartupTimeTests.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.metric.TorHiddenServiceStartupTime; - -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import java.io.File; - -import java.util.Map; -import java.util.Properties; - -import org.junit.Assert; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import static com.google.common.base.Preconditions.checkNotNull; - -@Disabled // Ignore for normal test runs as the tests take lots of time -public class TorHiddenServiceStartupTimeTests { - - private final static File torWorkingDirectory = new File("monitor/" + TorHiddenServiceStartupTimeTests.class.getSimpleName()); - - /** - * A dummy Reporter for development purposes. - */ - private class DummyReporter extends Reporter { - - private long result; - - @Override - public void report(long value) { - result = value; - } - - public long results() { - return result; - } - - @Override - public void report(Map values) { - report(Long.parseLong(values.values().iterator().next())); - } - - @Override - public void report(Map values, String prefix) { - report(values); - } - - @Override - public void report(String key, String value, String timestamp, String prefix) { - - } - - @Override - public void report(long value, String prefix) { - report(value); - } - } - - @BeforeAll - public static void setup() throws TorCtlException { - // simulate the tor instance available to all metrics - Tor.setDefault(new NativeTor(torWorkingDirectory)); - } - - @Test - public void run() throws Exception { - DummyReporter reporter = new DummyReporter(); - - // configure - Properties configuration = new Properties(); - configuration.put("TorHiddenServiceStartupTime.enabled", "true"); - configuration.put("TorHiddenServiceStartupTime.run.interval", "5"); - - Metric DUT = new TorHiddenServiceStartupTime(reporter); - // start - DUT.configure(configuration); - - // give it some time and then stop - Thread.sleep(180 * 1000); - Metric.haltAllMetrics(); - - // observe results - Assert.assertTrue(reporter.results() > 0); - } - - @AfterAll - public static void cleanup() { - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - tor.shutdown(); - torWorkingDirectory.delete(); - } -} diff --git a/monitor/src/test/java/bisq/monitor/TorRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/TorRoundTripTimeTests.java deleted file mode 100644 index b2091806a48..00000000000 --- a/monitor/src/test/java/bisq/monitor/TorRoundTripTimeTests.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.metric.TorRoundTripTime; - -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import java.io.File; - -import java.util.Map; -import java.util.Properties; - -import org.junit.Assert; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Test the round trip time metric against the hidden service of tor project.org. - * - * @author Florian Reimair - */ -@Disabled // Ignore for normal test runs as the tests take lots of time -public class TorRoundTripTimeTests { - - /** - * A dummy Reporter for development purposes. - */ - private class DummyReporter extends Reporter { - - private Map results; - - @Override - public void report(long value) { - Assert.fail(); - } - - public Map hasResults() { - return results; - } - - @Override - public void report(Map values) { - results = values; - } - - @Override - public void report(Map values, String prefix) { - report(values); - } - - @Override - public void report(String key, String value, String timestamp, String prefix) { - - } - - @Override - public void report(long value, String prefix) { - report(value); - } - } - - private static final File workingDirectory = new File(TorRoundTripTimeTests.class.getSimpleName()); - - @BeforeAll - public static void setup() throws TorCtlException { - // simulate the tor instance available to all metrics - Tor.setDefault(new NativeTor(workingDirectory)); - } - - @ParameterizedTest - @ValueSource(strings = {"default", "3", "4", "10"}) - public void run(String sampleSize) throws Exception { - DummyReporter reporter = new DummyReporter(); - - // configure - Properties configuration = new Properties(); - configuration.put("TorRoundTripTime.enabled", "true"); - configuration.put("TorRoundTripTime.run.interval", "2"); - if (!"default".equals(sampleSize)) - configuration.put("TorRoundTripTime.run.sampleSize", sampleSize); - // torproject.org hidden service - configuration.put("TorRoundTripTime.run.hosts", "http://expyuzz4wqqyqhjn.onion:80"); - - Metric DUT = new TorRoundTripTime(reporter); - // start - DUT.configure(configuration); - - // give it some time to start and then stop - Thread.sleep(100); - - Metric.haltAllMetrics(); - - // observe results - Map results = reporter.hasResults(); - Assert.assertFalse(results.isEmpty()); - Assert.assertEquals(results.get("sampleSize"), sampleSize.equals("default") ? "1" : sampleSize); - - Integer p25 = Integer.valueOf(results.get("p25")); - Integer p50 = Integer.valueOf(results.get("p50")); - Integer p75 = Integer.valueOf(results.get("p75")); - Integer min = Integer.valueOf(results.get("min")); - Integer max = Integer.valueOf(results.get("max")); - Integer average = Integer.valueOf(results.get("average")); - - Assert.assertTrue(0 < min); - Assert.assertTrue(min <= p25 && p25 <= p50); - Assert.assertTrue(p50 <= p75); - Assert.assertTrue(p75 <= max); - Assert.assertTrue(min <= average && average <= max); - } - - @AfterAll - public static void cleanup() { - Tor tor = Tor.getDefault(); - checkNotNull(tor, "tor must not be null"); - tor.shutdown(); - workingDirectory.delete(); - } -} diff --git a/monitor/src/test/java/bisq/monitor/TorStartupTimeTests.java b/monitor/src/test/java/bisq/monitor/TorStartupTimeTests.java deleted file mode 100644 index e72325c847a..00000000000 --- a/monitor/src/test/java/bisq/monitor/TorStartupTimeTests.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.monitor; - -import bisq.monitor.metric.TorStartupTime; - -import java.util.Map; -import java.util.Properties; - -import org.junit.Assert; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -@Disabled // Ignore for normal test runs as the tests take lots of time -public class TorStartupTimeTests { - - /** - * A dummy Reporter for development purposes. - */ - private class DummyReporter extends Reporter { - - private long result; - - @Override - public void report(long value) { - result = value; - } - - public long results() { - return result; - } - - @Override - public void report(Map values) { - report(Long.parseLong(values.values().iterator().next())); - } - - @Override - public void report(Map values, String prefix) { - report(values); - } - - @Override - public void report(String key, String value, String timestamp, String prefix) { - - } - - @Override - public void report(long value, String prefix) { - report(value); - } - } - - @Test - public void run() throws Exception { - - DummyReporter reporter = new DummyReporter(); - - // configure - Properties configuration = new Properties(); - configuration.put("TorStartupTime.enabled", "true"); - configuration.put("TorStartupTime.run.interval", "2"); - configuration.put("TorStartupTime.run.socksPort", "9999"); - - Metric DUT = new TorStartupTime(reporter); - // start - DUT.configure(configuration); - - // give it some time and then stop - Thread.sleep(15 * 1000); - Metric.haltAllMetrics(); - - // TODO Test fails due timing issue - // observe results - Assert.assertTrue(reporter.results() > 0); - } -} diff --git a/monitor/uninstall_collectd_debian.sh b/monitor/uninstall_collectd_debian.sh deleted file mode 100755 index dc63b4cc00d..00000000000 --- a/monitor/uninstall_collectd_debian.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -echo "[*] Stopping Bisq Server monitoring utensils" -echo ' ' -echo 'This script will not remove any configuration or binaries from the system. It just stops the services.' - -sleep 10 -sudo systemctl stop nginx -sudo systemctl stop collectd -sudo systemctl disable nginx -sudo systemctl disable collectd -echo "[*] Done!" diff --git a/settings.gradle b/settings.gradle index b9c33414f19..bd203e742e9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,7 +6,6 @@ include 'core' include 'cli' include 'daemon' include 'desktop' -include 'monitor' include 'seednode' include 'statsnode' include 'apitest' From 0e4255c61bd6c28ec3c3eebf8d23719e67da511c Mon Sep 17 00:00:00 2001 From: HenrikJannsen Date: Fri, 25 Nov 2022 12:37:02 -0500 Subject: [PATCH 3/4] Update verification-metadata.xml Remove classpath to springframework Signed-off-by: HenrikJannsen --- Makefile | 1 - build.gradle | 1 - gradle/verification-metadata.xml | 3375 ++---------------------------- 3 files changed, 153 insertions(+), 3224 deletions(-) diff --git a/Makefile b/Makefile index 1575383b664..c64dded96ab 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,6 @@ # # $ ls -1 bisq-* # bisq-desktop -# bisq-relay # bisq-seednode # bisq-statsnode # diff --git a/build.gradle b/build.gradle index 5c4ae044e06..1bc0d53c784 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,6 @@ buildscript { classpath 'com.google.gradle:osdetector-gradle-plugin:1.6.0' classpath 'com.github.jengelman.gradle.plugins:shadow:5.2.0' classpath 'org.openjfx:javafx-plugin:0.0.10' - classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.5.6' } } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 72cf949dc73..a4a07b525af 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -25,14 +25,6 @@ - - - - - - - - @@ -41,108 +33,26 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -151,38 +61,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -191,30 +69,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -223,123 +77,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -363,9 +100,6 @@ - - - @@ -375,68 +109,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -445,46 +122,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -493,46 +130,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -541,55 +138,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -598,38 +151,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -638,38 +159,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -678,38 +167,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -718,38 +175,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -758,30 +183,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -827,30 +228,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -869,248 +246,36 @@ - - - + + + + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -1131,14 +296,6 @@ - - - - - - - - @@ -1160,11 +317,6 @@ - - - - - @@ -1175,14 +327,6 @@ - - - - - - - - @@ -1204,14 +348,6 @@ - - - - - - - - @@ -1220,22 +356,6 @@ - - - - - - - - - - - - - - - - @@ -1244,19 +364,6 @@ - - - - - - - - - - - - - @@ -1267,26 +374,11 @@ - - - - - - - - - - - - - - - @@ -1295,66 +387,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1384,19 +416,6 @@ - - - - - - - - - - - - - @@ -1418,53 +437,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1509,86 +490,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1650,123 +551,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + @@ -1776,14 +569,6 @@ - - - - - - - - @@ -1792,14 +577,6 @@ - - - - - - - - @@ -1808,14 +585,6 @@ - - - - - - - - @@ -1824,14 +593,6 @@ - - - - - - - - @@ -1840,14 +601,6 @@ - - - - - - - - @@ -1856,14 +609,6 @@ - - - - - - - - @@ -1873,192 +618,13 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2066,174 +632,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2243,22 +641,6 @@ - - - - - - - - - - - - - - - - @@ -2267,30 +649,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -2315,14 +673,6 @@ - - - - - - - - @@ -2331,14 +681,6 @@ - - - - - - - - @@ -2347,11 +689,6 @@ - - - - - @@ -2370,27 +707,6 @@ - - - - - - - - - - - - - - - - - - - - - @@ -2407,32 +723,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2448,11 +738,6 @@ - - - - - @@ -2489,22 +774,6 @@ - - - - - - - - - - - - - - - - @@ -2521,22 +790,6 @@ - - - - - - - - - - - - - - - - @@ -2557,11 +810,6 @@ - - - - - @@ -2585,24 +833,11 @@ - - - - - - - - - - - - - @@ -2621,60 +856,16 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2683,29 +874,6 @@ - - - - - - - - - - - - - - - - - - - - - - - @@ -2714,54 +882,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -2794,1053 +914,205 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + - - - + + + - - + + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - - - + + - - - - - - + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -3870,15 +1142,9 @@ - - - - - - @@ -3887,15 +1153,9 @@ - - - - - - @@ -3904,29 +1164,17 @@ - - - - - - - - - - - - @@ -4003,30 +1251,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -4035,47 +1259,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -4091,247 +1279,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -4348,21 +1295,5 @@ - - - - - - - - - - - - - - - - From 3de6507a90537079a7871ef84dcdece9baf811c0 Mon Sep 17 00:00:00 2001 From: HenrikJannsen Date: Fri, 25 Nov 2022 15:24:03 -0500 Subject: [PATCH 4/4] Add readme with info where it is used Signed-off-by: HenrikJannsen --- statsnode/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 statsnode/README.md diff --git a/statsnode/README.md b/statsnode/README.md new file mode 100644 index 00000000000..5dde4f8bc3f --- /dev/null +++ b/statsnode/README.md @@ -0,0 +1 @@ +This module is used for bisq.markets.