diff --git a/.travis.yml b/.travis.yml index b6a0e40..3e88c4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ python: - "3.5" - "3.6" - "3.6-dev" # 3.6 development branch - - "3.7-dev" # 3.7 development branch +# - "3.7-dev" # 3.7 development branch install: - "pip install semantic_version" - "pip install ." @@ -14,6 +14,7 @@ after_success: - "pip install python-coveralls" - "coveralls" deploy: + skip_cleanup: true provider: pypi user: lockefox password: diff --git a/MANIFEST.in b/MANIFEST.in index e69de29..ce73eb3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -0,0 +1 @@ +include prosper/datareader/version.txt \ No newline at end of file diff --git a/README.rst b/README.rst index 69ad915..819f01f 100644 --- a/README.rst +++ b/README.rst @@ -40,8 +40,15 @@ Supported Feeds - Market News Feed: Google - Price Quote: Robinhood +**Coins**: helper libraries for fetching info on crypto currencies (via `hitBTC`_) + +- Ticker Info: get info about coin<->currency conversion metadata +- Price Quote: get latest OHLC data for given coin +- Order Book: view current orders + .. _pandas-datareader: https://pandas-datareader.readthedocs.io/en/latest/index.html .. _vader_sentiment: http://www.nltk.org/api/nltk.sentiment.html#module-nltk.sentiment.vader +.. _hitBTC: https://hitbtc.com .. |Show Logo| image:: http://dl.eveprosper.com/podcast/logo-colour-17_sm2.png :target: http://eveprosper.com diff --git a/docs/coins_help.rst b/docs/coins_help.rst new file mode 100644 index 0000000..52187e9 --- /dev/null +++ b/docs/coins_help.rst @@ -0,0 +1,72 @@ +======================== +prosper.datareader.coins +======================== + +Meant as an extension of `pandas-datareader`_, ``prosper.datareader.coins`` provides the ability to fetch and parse data pertaining to crypto currencies. + +``prosper.datareader.coins`` relies on services from `hitBTC`_. + +Info +==== + +General metadata and feed testing tools. + +**NOTE**: will implement caching layer for info, since this data should only refresh daily + +get_symbol() +------------ + + ``symbol_name = coins.info.get_symbol('COIN_TIKER', 'CONVERT_TICKER')`` + +Price of a crypto currency is measured in relation to other currencies a la FOREX. `hitBTC`_ requires a smash-cut version of coin + currency. + +Examples: + ++------+----------+--------+ +| Coin | Currency | Ticker | ++======+==========+========+ +| BTC | USD | BTCUSD | ++------+----------+--------+ +| ETH | EUR | ETHEUR | ++------+----------+--------+ +| ETH | BTC | ETHBTC | ++------+----------+--------+ + +Expected supported currencies: + +- ``USD`` +- ``EUR`` +- ``ETH`` +- ``BTC`` + +For more info, try ``info.supported_currencies()`` for a current list + +get_ticker_info() +----------------- + + ``ticker_info = coins.info.get_ticker_info('TICKER')`` + +If working backwards from a ticker, this function returns the original `hitBTC symbols`_ data. + +Prices +====== + +get_quote_hitbtc() +------------------ + + ``quote_df = coins.prices.get_quote_hitbtc(['BTC', 'ETH'])`` + +Get a peek at the current price and trend of your favorite crypto currency. This feed helps get OHLC data as well as mimic `pandas-datareader`_ quote behavior with keys like ``pct_change``. + +get_orderbook_hitbtc() +---------------------- + + ``orderbook = coins.prices.get_orderbook_hitbtc('BTC', 'asks')`` + +When you absolutely, positively, need all the data... go to the orderbook. This supports ``asks`` and ``bids`` for lookup. + +## TODO: add ``both`` behavior + +.. _pandas-datareader: https://pandas-datareader.readthedocs.io/en/latest/index.html +.. _hitBTC: https://hitbtc.com/ +.. _hitBTC symbols: https://hitbtc.com/api#symbols \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index df44587..8e8cd40 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,7 +23,7 @@ Supported Feeds * `Utils`_: General utilities for additional insights * `Stocks`_: Parse IRL stock quote data - +* `Coins`_: Data utilities for cryptocoin price quotes Index ===== @@ -33,9 +33,10 @@ Index :caption: Contents: getting_started.rst - utils_help.rst stocks_help.rst - + coins_help.rst + utils_help.rst + Indices and tables ================== @@ -47,6 +48,7 @@ Indices and tables .. _pandas-datareader: https://pandas-datareader.readthedocs.io/en/latest/index.html .. _Stocks: stocks_help.html .. _Utils: utils_help.html +.. _Coins: coins_help.html .. |Build Status| image:: https://travis-ci.org/EVEprosper/ProsperDatareader.svg?branch=master :target: https://travis-ci.org/EVEprosper/ProsperDatareader diff --git a/docs/source/datareader.coins.rst b/docs/source/datareader.coins.rst new file mode 100644 index 0000000..590df7d --- /dev/null +++ b/docs/source/datareader.coins.rst @@ -0,0 +1,30 @@ +datareader\.coins package +========================= + +Submodules +---------- + +datareader\.coins\.info module +------------------------------ + +.. automodule:: datareader.coins.info + :members: + :undoc-members: + :show-inheritance: + +datareader\.coins\.prices module +-------------------------------- + +.. automodule:: datareader.coins.prices + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: datareader.coins + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/datareader.rst b/docs/source/datareader.rst index 48c30ab..8d49a95 100644 --- a/docs/source/datareader.rst +++ b/docs/source/datareader.rst @@ -6,6 +6,7 @@ Subpackages .. toctree:: + datareader.coins datareader.stocks Submodules diff --git a/prosper/datareader/_version.py b/prosper/datareader/_version.py index cfa50fe..c79ed60 100644 --- a/prosper/datareader/_version.py +++ b/prosper/datareader/_version.py @@ -18,11 +18,15 @@ def get_version(): """ if not INSTALLED: - warnings.warn( - 'Unable to resolve package version until installed', - UserWarning - ) - return '0.0.0' #can't parse version without stuff installed + try: + with open('version.txt', 'r') as v_fh: + return v_fh.read() + except Exception: + warnings.warn( + 'Unable to resolve package version until installed', + UserWarning + ) + return '0.0.0' #can't parse version without stuff installed return p_version.get_version(HERE) diff --git a/prosper/datareader/coins/__init__.py b/prosper/datareader/coins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prosper/datareader/coins/info.py b/prosper/datareader/coins/info.py new file mode 100644 index 0000000..7959d29 --- /dev/null +++ b/prosper/datareader/coins/info.py @@ -0,0 +1,117 @@ +"""datareader.coins.info.py: tools for fetching cryptocoin metadata""" +from datetime import datetime +import itertools +from os import path + +import requests +import pandas as pd + +from prosper.datareader.config import LOGGER as G_LOGGER +import prosper.datareader.exceptions as exceptions + +LOGGER = G_LOGGER +HERE = path.abspath(path.dirname(__file__)) + +__all__ = ( + 'get_symbol', 'get_ticker_info', 'supported_symbol_info' +) + +SYMBOLS_URI = 'http://api.hitbtc.com/api/1/public/symbols' +def get_supported_symbols_hitbtc( + uri=SYMBOLS_URI, + data_key='symbols' +): + """fetch supported symbols from API + + Note: + Supported by hitbtc + https://hitbtc.com/api#symbols + + Args: + uri (str, optional): address for API + data_key (str, optional): data key name in JSON data + + Returns: + (:obj:`list`): list of supported feeds + + """ + req = requests.get(uri) + req.raise_for_status() + data = req.json() + + return data[data_key] + +################################################################################ + +def supported_symbol_info( + key_name +): + """find unique values for key_name in symbol feed + + Args: + key_name (str): name of key to search + + Returns: + (:obj:`list`): list of unique values + + """ + symbols_df = pd.DataFrame(get_supported_symbols_hitbtc()) + + unique_list = list(symbols_df[key_name].unique()) + + return unique_list + +def get_symbol( + commodity_ticker, + currency_ticker, + logger=LOGGER +): + """get valid ticker to look up + + Args: + commodity_ticker (str): short-name for crypto coin + currency_ticker (str): short-name for currency + logger (:obj:`logging.logger`, optional): logging handle + + Returns: + (str): valid ticker for HITBTC + + """ + logger.info('--Fetching symbol list from API') + symbols_df = pd.DataFrame(get_supported_symbols_hitbtc()) + + symbol = symbols_df.query( + 'commodity==\'{commodity}\' & currency==\'{currency}\''.format( + commodity=commodity_ticker.upper(), + currency=currency_ticker.upper() + )) + + if symbol.empty: + raise exceptions.SymbolNotSupported() + + return symbol['symbol'].iloc[0] + +def get_ticker_info( + ticker, + logger=LOGGER +): + """reverse lookup, get more info about a requested ticker + + Args: + ticker (str): info ticker for coin (ex: BTCUSD) + force_refresh (bool, optional): ignore local cacne and fetch directly from API + logger (:obj:`logging.logger`, optional): logging handle + + Returns: + (:obj:`dict`): hitBTC info about requested ticker + + """ + logger.info('--Fetching symbol list from API') + data = get_supported_symbols_hitbtc() + + ## Skip pandas, vanilla list search ok here + for ticker_info in data: + if ticker_info['symbol'] == ticker.upper(): + return ticker_info + + raise exceptions.TickerNotFound() diff --git a/prosper/datareader/coins/prices.py b/prosper/datareader/coins/prices.py new file mode 100644 index 0000000..8576903 --- /dev/null +++ b/prosper/datareader/coins/prices.py @@ -0,0 +1,227 @@ +"""datareader.coins.prices.py: tools for fetching cryptocoin price data""" +from datetime import datetime +from os import path +from enum import Enum + +import requests +import pandas as pd + +import prosper.datareader.config as config +import prosper.datareader.exceptions as exceptions +import prosper.datareader.coins.info as info + +LOGGER = config.LOGGER +HERE = path.abspath(path.dirname(__file__)) + +__all__ = ('get_orderbook_hitbtc', 'get_quote_hitbtc') + + +class OrderBook(Enum): + """enumerator for handling order book info""" + asks = 'asks' + bids = 'bids' + +def _listify( + data, + key_name +): + """recast data from dict to list, compress keys into sub-dict + + Args: + data (:obj:`dict`): data to transform (dict(dict)) + key_name (str): name to recast key to + + Returns: + (:obj:`list`): fixed data + + """ + listified_data = [] + for key, value in data.items(): + row = dict(value) + row[key_name] = key + listified_data.append(row) + + return listified_data + +def coin_list_to_symbol_list( + coin_list, + currency='USD', + strict=False +): + """convert list of crypto currencies to HitBTC symbols + + Args: + coin_list (str or :obj:`list`): list of coins to convert + currency (str, optional): currency to FOREX against + strict (bool, optional): throw if unsupported ticker is requested + + Returns: + (:obj:`list`): list of valid coins and tickers + + """ + valid_symbol_list = info.supported_symbol_info('symbol') + + symbols_list = [] + invalid_symbols = [] + for coin in coin_list: + ticker = str(coin).upper() + currency + if ticker not in valid_symbol_list: + invalid_symbols.append(ticker) + + symbols_list.append(ticker) + + if invalid_symbols and strict: + raise KeyError('Unsupported ticker requested: {}'.format(invalid_symbols)) + + return symbols_list + +COIN_TICKER_URI = 'http://api.hitbtc.com/api/1/public/{symbol}/ticker' +def get_ticker_hitbtc( + symbol, + uri=COIN_TICKER_URI +): + """fetch quote for coin + + Notes: + incurs a .format(ticker=symbol) call, be careful with overriding uri + + Args: + symbol (str): name of coin-ticker to pull + uri (str, optional): resource link + + Returns: + (:obj:`dict`) or (:obj:`list`): ticker data for desired coin + + """ + full_uri = '' + if not symbol: + ## fetching entire ticker list ## + full_uri = uri.replace(r'{symbol}/', '') + else: + full_uri = uri.format(symbol=symbol) + + req = requests.get(full_uri) + req.raise_for_status() + data = req.json() + + if not symbol: + ## fetching entire ticker list ## + data = _listify(data, 'symbol') + + return data + +COIN_ORDER_BOOK_URI = 'http://api.hitbtc.com/api/1/public/{symbol}/orderbook' +def get_order_book_hitbtc( + symbol, + format_price='number', + format_amount='number', + uri=COIN_ORDER_BOOK_URI +): + """fetch orderbook data + + Notes: + incurs a .format(ticker=symbol) call, be careful with overriding uri + + Args: + symbol (str): name of coin-ticker to pull + format_price (str, optional): optional format helper + format_amount (str, optional): optional format helper + uri (str, optional): resource link + + Returns: + (:obj:`dict`): order book for coin-ticker + + """ + params = {} + #TODO: this sucks + if format_price: + params['format_price'] = format_price + if format_amount: + params['format_amount'] = format_amount + + full_uri = uri.format(symbol=symbol) + req = requests.get(full_uri, params=params) + req.raise_for_status() + + data = req.json() + + return data #return both bids/asks for other steps to clean up later + +################################################################################ + +def get_quote_hitbtc( + coin_list, + currency='USD', + logger=LOGGER +): + """fetch common summary data for crypto-coins + + Args: + coin_list (:obj:`list`): list of tickers to look up' + currency (str, optional): currency to FOREX against + logger (:obj:`logging.logger`, optional): logging handle + + Returns: + (:obj:`pandas.DataFrame`): coin info for the day, JSONable + + """ + logger.info('Generating quote for %s -- HitBTC', config._list_to_str(coin_list)) + + logger.info('--validating coin_list') + ticker_list = coin_list_to_symbol_list( + coin_list, + currency=currency, + strict=True + ) + + logger.info('--fetching ticker data') + raw_quote = get_ticker_hitbtc('') + quote_df = pd.DataFrame(raw_quote) + + logger.info('--filtering ticker data') + quote_df = quote_df[quote_df['symbol'].isin(ticker_list)] + quote_df = quote_df[list(quote_df.columns.values)].apply(pd.to_numeric, errors='ignore') + quote_df['pct_change'] = (quote_df['last'] - quote_df['open']) / quote_df['open'] * 100 + + logger.debug(quote_df) + return quote_df + +def get_orderbook_hitbtc( + coin, + which_book, + currency='USD', + logger=LOGGER +): + """fetch current orderbook from hitBTC + + Args: + coin (str): name of coin to fetch + which_book (str): Enum, 'asks' or 'bids' + currency (str, optional): currency to FOREX against + + logger (:obj:`logging.logger`, optional): logging handle + + Returns: + (:obj:`pandas.DataFrame`): current coin order book + + """ + logger.info('Generating orderbook for %s -- HitBTC', coin) + order_enum = OrderBook(which_book) # validates which order book key to use + + logger.info('--validating coin') + symbol = coin_list_to_symbol_list( + [coin], + currency=currency, + strict=True + )[0] + + logger.info('--fetching orderbook') + raw_orderbook = get_order_book_hitbtc(symbol)[which_book] + + orderbook_df = pd.DataFrame(raw_orderbook, columns=['price', 'ammount']) + orderbook_df['symbol'] = symbol + orderbook_df['coin'] = coin + orderbook_df['orderbook'] = which_book + + logger.debug(orderbook_df) + return orderbook_df diff --git a/prosper/datareader/config.py b/prosper/datareader/config.py index 3e404b7..eca8f57 100644 --- a/prosper/datareader/config.py +++ b/prosper/datareader/config.py @@ -1,3 +1,20 @@ import prosper.common.prosper_logging as p_logging LOGGER = p_logging.DEFAULT_LOGGER + +def _list_to_str(ticker_list): + """parses/joins ticker list + + Args: + ticker_list (:obj:`list` or str): ticker(s) to parse + + Returns: + (str): list of tickers + + """ + if isinstance(ticker_list, str): + return ticker_list.upper() + elif isinstance(ticker_list, list): + return ','.join(ticker_list).upper() + else: + raise TypeError diff --git a/prosper/datareader/exceptions.py b/prosper/datareader/exceptions.py index b674f0e..38d6d1b 100644 --- a/prosper/datareader/exceptions.py +++ b/prosper/datareader/exceptions.py @@ -32,3 +32,16 @@ class StocksPricesException(StocksException): class StocksNewsException(StocksException): """base class for Datareader.stocks.prices""" pass + +########### +## Coins ## +########### +class CoinsException(DatareaderException): + """base class for Datareader.coins""" + pass +class SymbolNotSupported(CoinsException): + """symbol not supported by API source""" + pass +class TickerNotFound(CoinsException): + """unable to find more information about requested ticker""" + pass diff --git a/prosper/datareader/stocks/prices.py b/prosper/datareader/stocks/prices.py index b44aeb2..a5f2dda 100644 --- a/prosper/datareader/stocks/prices.py +++ b/prosper/datareader/stocks/prices.py @@ -9,27 +9,11 @@ import requests import pandas as pd -from prosper.datareader.config import LOGGER as G_LOGGER +import prosper.datareader.config as config -LOGGER = G_LOGGER +LOGGER = config.LOGGER HERE = path.abspath(path.dirname(__file__)) -def ticker_list_to_str(ticker_list): - """parses/joins ticker list - - Args: - ticker_list (:obj:`list` or str): ticker(s) to parse - - Returns: - (str): list of tickers - - """ - if isinstance(ticker_list, str): - return ticker_list.upper() - elif isinstance(ticker_list, list): - return ','.join(ticker_list).upper() - else: - raise TypeError def cast_str_to_int(dataframe): """tries to apply to_numeric to each str column @@ -70,7 +54,7 @@ def fetch_price_quotes_rh( (:obj:`list`): results from endpoint, JSONable """ - ticker_list_str = ticker_list_to_str(ticker_list) + ticker_list_str = config._list_to_str(ticker_list) logger.info('fetching quote data for %s -- Robinhood', ticker_list_str) params = { @@ -202,7 +186,7 @@ def get_quote_rh( {'ticker', 'company_name', 'price', 'percent_change', 'PE', 'short_ratio', 'quote_datetime'} """ - logger.info('Generating quote for %s -- Robinhood', ticker_list_to_str(ticker_list)) + logger.info('Generating quote for %s -- Robinhood', config._list_to_str(ticker_list)) ## Gather Required Data ## summary_raw_data = [] diff --git a/prosper/datareader/utils.py b/prosper/datareader/utils.py index 971d9b8..4572776 100644 --- a/prosper/datareader/utils.py +++ b/prosper/datareader/utils.py @@ -13,7 +13,9 @@ from prosper.datareader.config import LOGGER as G_LOGGER import prosper.datareader.exceptions as exceptions +HERE = path.abspath(path.dirname(__file__)) _TESTMODE = False + INSTALLED_PACKAGES = [] def _validate_install( package_name, diff --git a/setup.py b/setup.py index 02acfb2..09b37bc 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,8 @@ def initialize_options(self): packages=hack_find_packages('prosper'), include_package_data=True, package_data={ - '': ['LICENSE', 'README.rst'] + '': ['LICENSE', 'README.rst'], + 'prosper': ['datareader/version.txt'] }, install_requires=[ 'prospercommon', @@ -138,8 +139,7 @@ def initialize_options(self): 'requests', 'six', 'semantic-version', - 'demjson~=2.2.4', - 'ujson~=1.35' + 'demjson~=2.2.4' ], tests_require=[ 'jsonschema', diff --git a/tests/schemas/coins/hitbtc_orderbook.schema b/tests/schemas/coins/hitbtc_orderbook.schema new file mode 100644 index 0000000..1639588 --- /dev/null +++ b/tests/schemas/coins/hitbtc_orderbook.schema @@ -0,0 +1,25 @@ +{ + "type": "object", + "properties":{ + "asks":{ + "type": "array", + "items": { + "type": "array", + "items": { + "type":"number" + } + } + }, + "bids":{ + "type": "array", + "items": { + "type": "array", + "items": { + "type":"number" + } + } + } + }, + "required": ["asks", "bids"], + "additionalProperties": false +} \ No newline at end of file diff --git a/tests/schemas/coins/hitbtc_symbols.schema b/tests/schemas/coins/hitbtc_symbols.schema new file mode 100644 index 0000000..e873b43 --- /dev/null +++ b/tests/schemas/coins/hitbtc_symbols.schema @@ -0,0 +1,14 @@ +{ + "type": "object", + "properties":{ + "symbol": {"type":"string", "pattern":"([A-Z])+"}, + "step": {"type":"string"}, + "lot": {"type":"string"}, + "currency": {"type":"string", "pattern":"([A-Z])+"}, + "commodity": {"type":"string", "pattern":"([A-Z])+"}, + "takeLiquidityRate": {"type":"string"}, + "provideLiquidityRate": {"type":"string"} + }, + "required": ["symbol", "currency", "commodity"], + "additionalProperties": false +} \ No newline at end of file diff --git a/tests/schemas/coins/hitbtc_ticker.schema b/tests/schemas/coins/hitbtc_ticker.schema new file mode 100644 index 0000000..5b460dc --- /dev/null +++ b/tests/schemas/coins/hitbtc_ticker.schema @@ -0,0 +1,20 @@ +{ + "type": "object", + "properties":{ + "ask": {"type":"string"}, + "bid": {"type":"string"}, + "last": {"type":"string"}, + "low": {"type":"string"}, + "high": {"type":"string"}, + "open": {"type":"string"}, + "volume": {"type":"string"}, + "volume_quote": {"type":"string"}, + "timestamp": {"type":"integer"}, + "symbol": {"type":"string"} + }, + "required": [ + "ask", "bid", "last", "low", "high", "open", "volume", + "volume_quote", "timestamp" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/tests/schemas/stocks/rh_news.schema b/tests/schemas/stocks/rh_news.schema index 63a0b97..5863448 100644 --- a/tests/schemas/stocks/rh_news.schema +++ b/tests/schemas/stocks/rh_news.schema @@ -11,11 +11,13 @@ "updated_at": {"type":"string"}, "instrument": {"type":"string", "format":"uri"}, "num_clicks": {"type":"integer"}, - "uuid": {"type":"null"} + "uuid": {"type":["string", "null"]}, + "relay_url": {"type":"string", "format":"date-time"}, + "preview_image_url": {"type":"string"} }, "requires": [ "url", "title", "source", "published_at", "author", "summary", "api_source", "updated_at", "instrument", - "num_clicks", "uuid"], + "num_clicks", "uuid", "relay_url", "preview_image_url"], "additionalProperties": false } \ No newline at end of file diff --git a/tests/test_coins_info.py b/tests/test_coins_info.py new file mode 100644 index 0000000..03f1d3d --- /dev/null +++ b/tests/test_coins_info.py @@ -0,0 +1,107 @@ +"""test_coins_info.py: validate behavior for datareader.coins.info""" +from datetime import datetime +from os import path + +import pytest +import requests +import helpers +import pandas + +import prosper.datareader.coins.info as info +import prosper.datareader.exceptions as exceptions + +def test_get_supported_symbols_hitbtc(): + """validate get_supported_symbols_hitbtc() returns valid schema""" + data = info.get_supported_symbols_hitbtc() + + for symbol in data: + helpers.validate_schema( + symbol, + path.join('coins', 'hitbtc_symbols.schema') + ) + +class TestSupportedSymbolInfo: + """validate supported_symbol_info() behavior""" + def test_supported_commodities(self): + """validate commoddity list""" + commodity_list = info.supported_symbol_info('commodity') + + assert isinstance(commodity_list, list) + + expected_commodities = [ + 'BCN', 'BTC', 'DASH', 'DOGE', 'DSH', 'EMC', 'ETH', 'FCN', 'LSK', 'LTC', + 'NXT', 'QCN', 'SBD', 'SC', 'STEEM', 'XDN', 'XEM', 'XMR', 'ARDR', 'ZEC', + 'WAVES', 'MAID', 'AMP', 'BUS', 'DGD', 'ICN', 'SNGLS', '1ST', 'XLC', 'TRST', + 'TIME', 'GNO', 'REP', 'ZRC', 'BOS', 'DCT', 'AEON', 'GUP', 'PLU', 'LUN', + 'TAAS', 'NXC', 'EDG', 'RLC', 'SWT', 'TKN', 'WINGS', 'XAUR', 'AE', 'PTOY', + 'WTT', 'ETC', 'CFI', 'PLBT', 'BNT', 'XDNCO', 'FYN', 'SNM', 'SNT', 'CVC', + 'PAY', 'OAX', 'OMG', 'BQX', 'XTZ', 'CRS', 'DICE', 'XRP', 'MPK', 'NET', + 'STRAT', 'SNC', 'ADX', 'BET', 'EOS', 'DENT', 'SAN', 'MNE', 'MRV', 'MSP', + 'DDF', 'UET', 'MYB', 'SUR', 'IXT', 'HRB', 'PLR', 'TIX', 'NDC', 'PRO', + 'AVT', 'TFL', 'COSS', 'PBKX', 'PQT', '8BT', 'EVX', 'IML', 'ROOTS', 'DELTA', + 'QAU', 'MANA', 'DNT', 'FYP', 'OPT', 'GRPH', 'TNT', 'STX', 'CAT', 'BCC', + 'ECAT', 'BAS', 'ZRX', 'RVT', 'ICOS', 'PPC', 'VERI', 'IGNIS', 'QTUM', + 'PRG', 'BMC', 'CND', 'ANT', 'EMGO', 'SKIN', 'FUN', 'HVN', 'AMB', 'CDT', + 'AIR', 'POE', 'FUEL', 'MCAP' + ] + unique_commodities = list(set(commodity_list) - set(expected_commodities)) + missing_commodities = list(set(expected_commodities) - set(commodity_list)) + print('missing_commodities={}'.format(missing_commodities)) + + assert missing_commodities == [] + if unique_commodities: + pytest.xfail('unique_commodities={}'.format(unique_commodities)) + + def test_supported_currencies(self): + """validate currency list""" + currency_list = info.supported_symbol_info('currency') + + assert isinstance(currency_list, list) + + expected_currencies = ['BTC', 'EUR', 'USD', 'ETH'] + assert currency_list == expected_currencies + + def test_supported_symbols(self): + """validate symbols list""" + symbols_list = info.supported_symbol_info('symbol') + + assert isinstance(symbols_list, list) + + #TODO: validate values? + + +class TestGetSymbol: + """tests for info.get_symbol()""" + good_commodity = 'BTC' + good_currency = 'USD' + good_symbol = 'BTCUSD' + + def test_get_symbol_happypath_nocache(self): + """validate expected behavior for get_symbol()""" + symbol = info.get_symbol(self.good_commodity, self.good_currency) + + assert isinstance(symbol, str) + assert symbol == self.good_symbol + + def test_get_symbol_bad_symbol(self): + """validate exception case when bad symbol inputs requested""" + with pytest.raises(exceptions.SymbolNotSupported): + bad_symbol = info.get_symbol('BUTTS', self.good_currency) + + with pytest.raises(exceptions.SymbolNotSupported): + bad_symbol = info.get_symbol(self.good_commodity, 'BUTTS') + +class TestGetTickerInfo: + """tests info.get_ticker_info()""" + good_symbol = 'BTCUSD' + + def test_get_ticker_info_happypath_nocache(self): + """validate expected behavior for get_ticker_info()""" + symbol_info = info.get_ticker_info(self.good_symbol) + assert symbol_info['symbol'] == self.good_symbol + assert symbol_info['currency'] == 'USD' + + def test_get_ticker_info_bad_symbol(self): + """validate exception case when bad symbol inputs requested""" + with pytest.raises(exceptions.TickerNotFound): + bad_symbol = info.get_ticker_info('BUTTS') diff --git a/tests/test_coins_prices.py b/tests/test_coins_prices.py new file mode 100644 index 0000000..47adc61 --- /dev/null +++ b/tests/test_coins_prices.py @@ -0,0 +1,180 @@ +"""test_coins_prices.py: validate behavior for datareader.coins.prices""" +from datetime import datetime +from os import path + +import pytest +import requests +import helpers +import pandas + +import prosper.datareader.coins.prices as prices +import prosper.datareader.exceptions as exceptions + +def test_listify(): + """validate expected behavior for _listify()""" + demo_data = { + 'key1': { + 'val1': 1, + 'val2': 2 + }, + 'key2': { + 'val1': 10, + 'val2': 20 + } + } + fixed_data = prices._listify(demo_data, 'key') + assert isinstance(fixed_data, list) + expected_keys = ['val1', 'val2', 'key'] + expected_keys.sort() + for row in fixed_data: + keys = list(row.keys()) + keys.sort() + assert keys == expected_keys + +def test_get_ticker_single(): + """validate get_ticker_hitbtc() returns valid schema""" + data = prices.get_ticker_hitbtc('BTCUSD') + + assert isinstance(data, dict) + helpers.validate_schema( + data, + path.join('coins', 'hitbtc_ticker.schema') + ) + + with pytest.raises(requests.exceptions.HTTPError): + bad_data = prices.get_ticker_hitbtc('BUTTS') + +def test_get_ticker_all(): + """validate get_ticker() behavior with blank args""" + data = prices.get_ticker_hitbtc('') + + assert isinstance(data, list) + helpers.validate_schema( + data[0], + path.join('coins', 'hitbtc_ticker.schema') + ) + assert len(data) >= 190 * 0.9 #expect ~same data or more + +def test_get_orderbook(): + """validate get_order_book_hitbtc() returns valid schema""" + data = prices.get_order_book_hitbtc('BTCUSD') + + assert isinstance(data, dict) + assert isinstance(data['asks'], list) + assert isinstance(data['bids'], list) + +def test_coin_list_to_symbol_list(): + """validate coin_list_to_symbol_list() works as expected""" + test_coin_list = ['BTC', 'ETH'] + + ticker_list = prices.coin_list_to_symbol_list(test_coin_list, currency='USD') + + expected_tickers = ['BTCUSD', 'ETHUSD'] + assert isinstance(ticker_list, list) + assert ticker_list == expected_tickers + + with pytest.raises(KeyError): + bad_ticker = prices.coin_list_to_symbol_list(['BUTTS'], strict=True) + +class TestGetQuoteHitBTC: + """validate get_quote_hitbtc() behavior""" + coin_list = ['BTC', 'ETH'] + bad_list = ['BUTTS'] + expected_headers = [ + 'ask', 'bid', 'high', 'last', 'low', 'open', 'symbol', + 'timestamp', 'volume', 'volume_quote', 'pct_change' + ] + + def test_get_quote_hitbtc_happypath(self): + """validate expected normal behavior""" + quote = prices.get_quote_hitbtc(self.coin_list) + + assert isinstance(quote, pandas.DataFrame) + + print(list(quote.columns.values)) + assert list(quote.columns.values) == self.expected_headers + assert len(quote) == len(self.coin_list) + + def test_get_quote_hitbtc_error(self): + """validate system throws as expected""" + with pytest.raises(KeyError): + bad_quote = prices.get_quote_hitbtc(self.bad_list) + + def test_get_quote_hitbtc_singleton(self): + """validate quote special case for 1 value""" + quote = prices.get_quote_hitbtc([self.coin_list[0]]) + + assert isinstance(quote, pandas.DataFrame) + + print(list(quote.columns.values)) + assert list(quote.columns.values) == self.expected_headers + assert len(quote) == 1 + +class TestGetOrderbookHitBTC: + """validate get_orderbook_hitbtc() behavior""" + coin = 'BTC' + expected_headers = ['price', 'ammount', 'symbol', 'coin', 'orderbook'] + + def test_get_orderbook_hitbtc_happypath_asks(self): + """validate expected normal behavior""" + asks_orderbook = prices.get_orderbook_hitbtc( + self.coin, + 'asks', + currency='USD' + ) + + assert isinstance(asks_orderbook, pandas.DataFrame) + print(list(asks_orderbook.columns.values)) + assert list(asks_orderbook.columns.values) == self.expected_headers + + orderbook = asks_orderbook['orderbook'].unique() + assert len(orderbook) == 1 + assert orderbook[0] == 'asks' + + coin = asks_orderbook['coin'].unique() + assert len(coin) == 1 + assert coin[0] == self.coin + + symbol = asks_orderbook['symbol'].unique() + assert len(symbol) == 1 + assert symbol[0] == self.coin + 'USD' + + def test_get_orderbook_hitbtc_happypath_bids(self): + """validate expected normal behavior""" + bids_orderbook = prices.get_orderbook_hitbtc( + self.coin, + 'bids', + currency='USD' + ) + + assert isinstance(bids_orderbook, pandas.DataFrame) + print(list(bids_orderbook.columns.values)) + assert list(bids_orderbook.columns.values) == self.expected_headers + + orderbook = bids_orderbook['orderbook'].unique() + assert len(orderbook) == 1 + assert orderbook[0] == 'bids' + + coin = bids_orderbook['coin'].unique() + assert len(coin) == 1 + assert coin[0] == self.coin + + symbol = bids_orderbook['symbol'].unique() + assert len(symbol) == 1 + assert symbol[0] == self.coin + 'USD' + + def test_get_orderbook_hitbtc_bad_book(self): + """validate expected error for asking for bad enum""" + with pytest.raises(ValueError): + bad_book = prices.get_orderbook_hitbtc( + self.coin, + 'butts' + ) + + def test_get_orderbook_hitbtc_bad_coin(self): + """validate expected error for asking for bad enum""" + with pytest.raises(KeyError): + bad_book = prices.get_orderbook_hitbtc( + 'BUTTS', + 'asks' + ) diff --git a/tests/test_stocks_news.py b/tests/test_stocks_news.py index 0ad6c98..22f3478 100644 --- a/tests/test_stocks_news.py +++ b/tests/test_stocks_news.py @@ -178,8 +178,9 @@ class TestCompanyNewsRobinhood: good_ticker = helpers.CONFIG.get('STOCKS', 'good_ticker') bad_ticker = helpers.CONFIG.get('STOCKS', 'bad_ticker') expected_news_cols = [ - 'api_source', 'author', 'instrument', 'num_clicks', 'published_at', - 'source', 'summary', 'title', 'updated_at', 'url', 'uuid' + 'api_source', 'author', 'instrument', 'num_clicks', 'preview_image_url', + 'published_at', 'relay_url', 'source', 'summary', 'title', 'updated_at', + 'url', 'uuid' ] @pytest.mark.long @@ -242,4 +243,5 @@ def test_vader_application(self): expected_cols = self.expected_news_cols expected_cols.extend(['neu', 'pos', 'compound', 'neg']) + print(list(graded_news.columns.values)) assert list(graded_news.columns.values) == expected_cols diff --git a/tests/test_stocks_prices.py b/tests/test_stocks_prices.py index 353bad3..b92f653 100644 --- a/tests/test_stocks_prices.py +++ b/tests/test_stocks_prices.py @@ -11,6 +11,7 @@ import prosper.datareader.stocks.prices as prices import prosper.datareader.exceptions as exceptions +import prosper.datareader.config as config class TestExpectedSchemas: good_ticker = helpers.CONFIG.get('STOCKS', 'good_ticker') @@ -68,24 +69,24 @@ def test_ticker_list_to_str(): """make sure ticker_list_to_str returns as expected""" no_caps_pattern = re.compile('[a-z]+') - single_stock = prices.ticker_list_to_str('MU') + single_stock = config._list_to_str('MU') assert not no_caps_pattern.match(single_stock) assert single_stock == 'MU' - lower_stock = prices.ticker_list_to_str('mu') + lower_stock = config._list_to_str('mu') assert not no_caps_pattern.match(lower_stock) assert lower_stock == 'MU' - multi_stock = prices.ticker_list_to_str(['MU', 'INTC', 'BA']) + multi_stock = config._list_to_str(['MU', 'INTC', 'BA']) assert not no_caps_pattern.match(multi_stock) assert multi_stock == 'MU,INTC,BA' - lower_multi_stock = prices.ticker_list_to_str(['MU', 'intc', 'BA']) + lower_multi_stock = config._list_to_str(['MU', 'intc', 'BA']) assert not no_caps_pattern.match(lower_multi_stock) assert lower_multi_stock == 'MU,INTC,BA' with pytest.raises(TypeError): - bad_stock = prices.ticker_list_to_str({'butts':1}) + bad_stock = config._list_to_str({'butts':1}) def test_cast_str_to_int(): """validate behavior for cast_str_to_int"""