From 4361bbfc648999ea2bc66381623a726cdfcb18b3 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 28 Jul 2019 22:06:56 +0100 Subject: [PATCH] BUG: Remove pandas.compat dependencies Remvoe dep on pandas.compat for 0.7.2 release --- docs/source/whatsnew/v0.7.0.txt | 1 + pandas_datareader/bankofcanada.py | 4 +-- pandas_datareader/base.py | 8 +++--- pandas_datareader/compat/__init__.py | 41 ++++++++++++++++++++++++---- pandas_datareader/enigma.py | 5 ++-- pandas_datareader/eurostat.py | 6 ++-- pandas_datareader/famafrench.py | 4 +-- pandas_datareader/google/quotes.py | 9 +++--- pandas_datareader/io/jsdmx.py | 10 +++---- pandas_datareader/io/sdmx.py | 8 +++--- pandas_datareader/io/util.py | 5 ++-- pandas_datareader/oecd.py | 4 +-- pandas_datareader/wb.py | 2 +- pandas_datareader/yahoo/fx.py | 4 +-- pandas_datareader/yahoo/quotes.py | 7 ++--- 15 files changed, 72 insertions(+), 46 deletions(-) diff --git a/docs/source/whatsnew/v0.7.0.txt b/docs/source/whatsnew/v0.7.0.txt index 339f9baf..31b3291f 100644 --- a/docs/source/whatsnew/v0.7.0.txt +++ b/docs/source/whatsnew/v0.7.0.txt @@ -64,6 +64,7 @@ Backwards incompatible API changes Bug Fixes ~~~~~~~~~ +- Fixed import of pandas.compat (:issue:`657`) - Added support for passing the API KEY to QuandlReader either directly or by setting the environmental variable QUANDL_API_KEY (:issue:`485`). - Added support for optionally passing a custom base_url to the EnigmaReader (:issue:`499`). diff --git a/pandas_datareader/bankofcanada.py b/pandas_datareader/bankofcanada.py index 6cb247b2..c8ca75b1 100644 --- a/pandas_datareader/bankofcanada.py +++ b/pandas_datareader/bankofcanada.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -import pandas.compat as compat +from pandas_datareader.compat import string_types from pandas_datareader.base import _BaseReader @@ -17,7 +17,7 @@ class BankOfCanadaReader(_BaseReader): @property def url(self): """API URL""" - if not isinstance(self.symbols, compat.string_types): + if not isinstance(self.symbols, string_types): raise ValueError('data name must be string') return '{0}/{1}/csv'.format(self._URL, self.symbols) diff --git a/pandas_datareader/base.py b/pandas_datareader/base.py index 47089c5f..c855c8ad 100644 --- a/pandas_datareader/base.py +++ b/pandas_datareader/base.py @@ -3,12 +3,12 @@ import numpy as np import requests +from io import StringIO -import pandas.compat as compat from pandas import DataFrame from pandas import read_csv, concat from pandas.io.common import urlencode -from pandas.compat import StringIO, bytes_to_str +from pandas_datareader.compat import bytes_to_str, string_types, binary_type from pandas_datareader._utils import (RemoteDataError, SymbolWarning, _sanitize_dates, _init_session) @@ -99,7 +99,7 @@ def _read_url_as_StringIO(self, url, params=None): service = self.__class__.__name__ raise IOError("{} request returned no data; check URL for invalid " "inputs: {}".format(service, self.url)) - if isinstance(text, compat.binary_type): + if isinstance(text, binary_type): out.write(bytes_to_str(text)) else: out.write(text) @@ -205,7 +205,7 @@ def _get_params(self, *args, **kwargs): def read(self): """Read data""" # If a single symbol, (e.g., 'GOOG') - if isinstance(self.symbols, (compat.string_types, int)): + if isinstance(self.symbols, (string_types, int)): df = self._read_one_data(self.url, params=self._get_params(self.symbols)) # Or multiple symbols, (e.g., ['GOOG', 'AAPL', 'MSFT']) diff --git a/pandas_datareader/compat/__init__.py b/pandas_datareader/compat/__init__.py index 46c68b78..e7b88f2b 100644 --- a/pandas_datareader/compat/__init__.py +++ b/pandas_datareader/compat/__init__.py @@ -1,12 +1,10 @@ # flake8: noqa +import pandas as pd +import pandas.io.common as com import sys from distutils.version import LooseVersion from io import BytesIO -import pandas as pd -import pandas.compat as compat -import pandas.io.common as com - PY3 = sys.version_info >= (3, 0) PANDAS_VERSION = LooseVersion(pd.__version__) @@ -45,8 +43,39 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None, else: from pandas.core.common import is_list_like - -if compat.PY3: +if PY3: from urllib.error import HTTPError + from functools import reduce + + string_types = str, + binary_type = bytes + + + def str_to_bytes(s, encoding=None): + return s.encode(encoding or 'ascii') + + + def bytes_to_str(b, encoding=None): + return b.decode(encoding or 'utf-8') else: from urllib2 import HTTPError + + reduce = reduce + binary_type = str + string_types = basestring, + + + def bytes_to_str(b, encoding=None): + return b + + + def str_to_bytes(s, encoding=None): + return s + + +def lmap(*args, **kwargs): + return list(map(*args, **kwargs)) + + +def lrange(*args, **kwargs): + return list(range(*args, **kwargs)) diff --git a/pandas_datareader/enigma.py b/pandas_datareader/enigma.py index 2b4a5c1a..0904a569 100644 --- a/pandas_datareader/enigma.py +++ b/pandas_datareader/enigma.py @@ -2,10 +2,9 @@ import time from io import StringIO -import pandas.compat as compat import pandas as pd -from pandas_datareader.base import _BaseReader +from pandas_datareader.base import _BaseReader, string_types class EnigmaReader(_BaseReader): @@ -65,7 +64,7 @@ def __init__(self, self._api_key = api_key self._dataset_id = dataset_id - if not isinstance(self._dataset_id, compat.string_types): + if not isinstance(self._dataset_id, string_types): raise ValueError( "The Enigma dataset_id must be a string (ex: " "'bedaf052-5fcd-4758-8d27-048ce8746c6a')") diff --git a/pandas_datareader/eurostat.py b/pandas_datareader/eurostat.py index 774c5c48..30414ae7 100644 --- a/pandas_datareader/eurostat.py +++ b/pandas_datareader/eurostat.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals import pandas as pd -import pandas.compat as compat from pandas_datareader.io.sdmx import read_sdmx, _read_sdmx_dsd +from pandas_datareader.compat import string_types from pandas_datareader.base import _BaseReader @@ -15,7 +15,7 @@ class EurostatReader(_BaseReader): @property def url(self): """API URL""" - if not isinstance(self.symbols, compat.string_types): + if not isinstance(self.symbols, string_types): raise ValueError('data name must be string') q = '{0}/data/{1}/?startperiod={2}&endperiod={3}' @@ -25,7 +25,7 @@ def url(self): @property def dsd_url(self): """API DSD URL""" - if not isinstance(self.symbols, compat.string_types): + if not isinstance(self.symbols, string_types): raise ValueError('data name must be string') return '{0}/datastructure/ESTAT/DSD_{1}'.format( diff --git a/pandas_datareader/famafrench.py b/pandas_datareader/famafrench.py index 87de9c0a..1da9ae80 100644 --- a/pandas_datareader/famafrench.py +++ b/pandas_datareader/famafrench.py @@ -5,7 +5,7 @@ from zipfile import ZipFile from pandas import read_csv, to_datetime -from pandas.compat import lmap +from pandas_datareader.compat import lmap from pandas_datareader.base import _BaseReader @@ -107,7 +107,7 @@ def _read_one_data(self, url, params): datasets, table_desc = {}, [] for i, src in enumerate(tables): - match = re.search('^\s*,', src, re.M) # the table starts there + match = re.search(r'^\s*,', src, re.M) # the table starts there start = 0 if not match else match.start() df = read_csv(StringIO('Date' + src[start:]), **params) diff --git a/pandas_datareader/google/quotes.py b/pandas_datareader/google/quotes.py index 75fd1b76..676d17e6 100644 --- a/pandas_datareader/google/quotes.py +++ b/pandas_datareader/google/quotes.py @@ -1,10 +1,11 @@ +import json +import numpy as np import pandas as pd +import re from dateutil.parser import parse -import numpy as np from pandas_datareader.base import _BaseReader -import json -import re +from pandas_datareader.compat import string_types class GoogleQuotesReader(_BaseReader): @@ -20,7 +21,7 @@ def url(self): @property def params(self): """Parameters to use in API calls""" - if isinstance(self.symbols, pd.compat.string_types): + if isinstance(self.symbols, string_types): sym_list = self.symbols else: sym_list = ','.join(self.symbols) diff --git a/pandas_datareader/io/jsdmx.py b/pandas_datareader/io/jsdmx.py index fb3da833..e993effb 100644 --- a/pandas_datareader/io/jsdmx.py +++ b/pandas_datareader/io/jsdmx.py @@ -1,13 +1,13 @@ # pylint: disable-msg=E1101,W0613,W0603 from __future__ import unicode_literals +from collections import OrderedDict import itertools import sys import numpy as np import pandas as pd -import pandas.compat as compat from pandas_datareader.io.util import _read_content @@ -38,7 +38,7 @@ def read_jsdmx(path_or_buf): if isinstance(jdata, dict): data = jdata else: - data = json.loads(jdata, object_pairs_hook=compat.OrderedDict) + data = json.loads(jdata, object_pairs_hook=OrderedDict) structure = data['structure'] index = _parse_dimensions(structure['dimensions']['observation']) @@ -56,9 +56,9 @@ def read_jsdmx(path_or_buf): def _get_indexer(index): if index.nlevels == 1: - return [str(i) for i in compat.range(len(index))] + return [str(i) for i in range(len(index))] else: - it = itertools.product(*[compat.range( + it = itertools.product(*[range( len(level)) for level in index.levels]) return [':'.join(map(str, i)) for i in it] @@ -68,7 +68,7 @@ def _parse_values(dataset, index, columns): series = dataset['series'] values = [] - # for s_key, s_value in compat.iteritems(series): + # for s_key, s_value in iteritems(series): for s_key in _get_indexer(columns): try: observations = series[s_key]['observations'] diff --git a/pandas_datareader/io/sdmx.py b/pandas_datareader/io/sdmx.py index 0e2395ea..4295c319 100644 --- a/pandas_datareader/io/sdmx.py +++ b/pandas_datareader/io/sdmx.py @@ -3,12 +3,12 @@ import collections import time import zipfile +from io import BytesIO import pandas as pd -import pandas.compat as compat from pandas_datareader.io.util import _read_content -from pandas_datareader.compat import HTTPError +from pandas_datareader.compat import HTTPError, str_to_bytes _STRUCTURE = '{http://www.sdmx.org/resources/sdmxml/schemas/v2_1/structure}' @@ -235,8 +235,8 @@ def _read_zipped_sdmx(path_or_buf): """ Unzipp data contains SDMX-XML """ data = _read_content(path_or_buf) - zp = compat.BytesIO() - zp.write(compat.str_to_bytes(data)) + zp = BytesIO() + zp.write(str_to_bytes(data)) f = zipfile.ZipFile(zp) files = f.namelist() assert len(files) == 1 diff --git a/pandas_datareader/io/util.py b/pandas_datareader/io/util.py index fedf07d3..25f6168e 100644 --- a/pandas_datareader/io/util.py +++ b/pandas_datareader/io/util.py @@ -2,8 +2,7 @@ import os -import pandas.compat as compat -from pandas_datareader.compat import get_filepath_or_buffer +from pandas_datareader.compat import get_filepath_or_buffer, string_types def _read_content(path_or_buf): @@ -13,7 +12,7 @@ def _read_content(path_or_buf): filepath_or_buffer = get_filepath_or_buffer(path_or_buf)[0] - if isinstance(filepath_or_buffer, compat.string_types): + if isinstance(filepath_or_buffer, string_types): try: exists = os.path.exists(filepath_or_buffer) except (TypeError, ValueError): diff --git a/pandas_datareader/oecd.py b/pandas_datareader/oecd.py index 134900dc..9d9d85e4 100644 --- a/pandas_datareader/oecd.py +++ b/pandas_datareader/oecd.py @@ -1,8 +1,8 @@ import pandas as pd -import pandas.compat as compat from pandas_datareader.io import read_jsdmx from pandas_datareader.base import _BaseReader +from pandas_datareader.compat import string_types class OECDReader(_BaseReader): @@ -15,7 +15,7 @@ def url(self): """API URL""" url = 'http://stats.oecd.org/SDMX-JSON/data' - if not isinstance(self.symbols, compat.string_types): + if not isinstance(self.symbols, string_types): raise ValueError('data name must be string') # API: https://data.oecd.org/api/sdmx-json-documentation/ diff --git a/pandas_datareader/wb.py b/pandas_datareader/wb.py index c7df8ee5..ecd583d7 100644 --- a/pandas_datareader/wb.py +++ b/pandas_datareader/wb.py @@ -2,7 +2,7 @@ import warnings -from pandas.compat import reduce, lrange, string_types +from pandas_datareader.compat import reduce, lrange, string_types import pandas as pd import numpy as np diff --git a/pandas_datareader/yahoo/fx.py b/pandas_datareader/yahoo/fx.py index 3200335d..adf264e7 100644 --- a/pandas_datareader/yahoo/fx.py +++ b/pandas_datareader/yahoo/fx.py @@ -3,8 +3,8 @@ import warnings from pandas import (DataFrame, Series, to_datetime, concat) from pandas_datareader.yahoo.daily import YahooDailyReader -import pandas.compat as compat from pandas_datareader._utils import (RemoteDataError, SymbolWarning) +from pandas_datareader.compat import string_types class YahooFXReader(YahooDailyReader): @@ -55,7 +55,7 @@ def read(self): """Read data""" try: # If a single symbol, (e.g., 'GOOG') - if isinstance(self.symbols, (compat.string_types, int)): + if isinstance(self.symbols, (string_types, int)): df = self._read_one_data(self.symbols) # Or multiple symbols, (e.g., ['GOOG', 'AAPL', 'MSFT']) diff --git a/pandas_datareader/yahoo/quotes.py b/pandas_datareader/yahoo/quotes.py index 2f14a1c7..b779e2ee 100644 --- a/pandas_datareader/yahoo/quotes.py +++ b/pandas_datareader/yahoo/quotes.py @@ -1,12 +1,9 @@ import json from collections import OrderedDict - -import pandas.compat as compat from pandas import DataFrame - from pandas_datareader.base import _BaseReader - +from pandas_datareader.compat import string_types _DEFAULT_PARAMS = { 'lang': 'en-US', @@ -24,7 +21,7 @@ def url(self): return 'https://query1.finance.yahoo.com/v7/finance/quote' def read(self): - if isinstance(self.symbols, compat.string_types): + if isinstance(self.symbols, string_types): return self._read_one_data(self.url, self.params(self.symbols)) else: data = OrderedDict()