From 5e90059af9615f892ddf97427e04d5f2fd97e273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Corbasson?= Date: Tue, 25 Aug 2020 11:28:01 +0200 Subject: [PATCH 001/163] Add a row_limit parameter to from_csv() --- agate/table/from_csv.py | 10 ++++++++-- tests/test_table/test_from_csv.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index 1e962a97..da897e25 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -1,11 +1,12 @@ #!/usr/bin/env python import io +import itertools import six @classmethod -def from_csv(cls, path, column_names=None, column_types=None, row_names=None, skip_lines=0, header=True, sniff_limit=0, encoding='utf-8', **kwargs): +def from_csv(cls, path, column_names=None, column_types=None, row_names=None, skip_lines=0, header=True, sniff_limit=0, encoding='utf-8', row_limit=None, **kwargs): """ Create a new table from a CSV. @@ -38,6 +39,8 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk Character encoding of the CSV file. Note: if passing in a file handle it is assumed you have already opened it with the correct encoding specified. + :param row_limit: + Limit how many rows of data will be read """ from agate import csv from agate.table import Table @@ -80,7 +83,10 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk else: next(reader) - rows = tuple(reader) + if row_limit is None: + rows = tuple(reader) + else: + rows = tuple(itertools.islice(reader, row_limit)) finally: if close: diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index 560813d8..a3b433fa 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -170,3 +170,21 @@ def test_from_csv_skip_lines_cr(self): self.assertColumnTypes(table2, [Number, Text, Boolean, Date, DateTime, TimeDelta]) self.assertRows(table2, table1.rows) + + def test_from_csv_row_limit(self): + table1 = Table(self.rows[:2], self.column_names, self.column_types) + table2 = Table.from_csv('examples/test.csv', row_limit=2) + + self.assertColumnNames(table2, table1.column_names) + self.assertColumnTypes(table2, [Number, Text, Boolean, Date, DateTime, TimeDelta]) + + self.assertRows(table2, table1.rows) + + def test_from_csv_row_limit_no_header_columns(self): + table1 = Table(self.rows[:2], self.column_names, self.column_types) + table2 = Table.from_csv('examples/test_no_header.csv', self.column_names, header=False, row_limit=2) + + self.assertColumnNames(table2, table1.column_names) + self.assertColumnTypes(table2, [Number, Text, Boolean, Date, DateTime, TimeDelta]) + + self.assertRows(table2, table1.rows) From dbfff1c849fd147e24d0da368f154a51a4bb345f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Corbasson?= Date: Tue, 25 Aug 2020 15:27:24 +0200 Subject: [PATCH 002/163] Check that, with a row_limit higher than the row count, we return the full table without any issue --- tests/test_table/test_from_csv.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index a3b433fa..9d500e2f 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -188,3 +188,12 @@ def test_from_csv_row_limit_no_header_columns(self): self.assertColumnTypes(table2, [Number, Text, Boolean, Date, DateTime, TimeDelta]) self.assertRows(table2, table1.rows) + + def test_from_csv_row_limit_too_high(self): + table1 = Table(self.rows, self.column_names, self.column_types) + table2 = Table.from_csv('examples/test.csv', row_limit=200) + + self.assertColumnNames(table2, table1.column_names) + self.assertColumnTypes(table2, [Number, Text, Boolean, Date, DateTime, TimeDelta]) + + self.assertRows(table2, table1.rows) From 93a28200b7e106b7185e0f53eaf4a71d6bac035f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Tue, 8 Sep 2020 01:16:51 +0200 Subject: [PATCH 003/163] Pin parsedatetime to <2.6 while waiting for upstream fixes --- requirements-py2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 9bc3a8d8..6a10474e 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -8,7 +8,7 @@ sphinx_rtd_theme>=0.1.6 wheel>=0.24.0 pytimeparse>=1.1.5 Babel>=2.0 -parsedatetime>=2.1 +parsedatetime>=2.1,<2.6 pytz>=2015.4 mock>=1.3.0 isodate>=0.5.4 From 72a946e4767ae6d2126e3d6e6fecad52e9876677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Mon, 7 Sep 2020 21:32:36 +0200 Subject: [PATCH 004/163] Add a parameter to force null_values in TypeTester() --- agate/type_tester.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/agate/type_tester.py b/agate/type_tester.py index e153fdb3..f1dd13b7 100644 --- a/agate/type_tester.py +++ b/agate/type_tester.py @@ -52,13 +52,26 @@ class TypeTester(object): options such as ``locale`` to :class:`.Number` or ``cast_nulls`` to :class:`.Text`. Take care in specifying the order of the list. It is the order they are tested in. :class:`.Text` should always be last. + :param null_values: + If :code:`types` is :code:`None`, a sequence of values which should be cast to + :code:`None` when encountered by the default data types. """ - def __init__(self, force={}, limit=None, types=None): + def __init__(self, force={}, limit=None, types=None, null_values=None): self._force = force self._limit = limit if types: self._possible_types = types + elif null_values: + # In order of preference + self._possible_types = [ + Boolean(null_values=null_values), + Number(null_values=null_values), + TimeDelta(null_values=null_values), + Date(null_values=null_values), + DateTime(null_values=null_values), + Text(null_values=null_values) + ] else: # In order of preference self._possible_types = [ From d934e5fe54b212b271c32b5fd50e46df411d4303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Tue, 8 Sep 2020 01:16:51 +0200 Subject: [PATCH 005/163] Pin parsedatetime to <2.6 while waiting for upstream fixes --- requirements-py2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 9bc3a8d8..6a10474e 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -8,7 +8,7 @@ sphinx_rtd_theme>=0.1.6 wheel>=0.24.0 pytimeparse>=1.1.5 Babel>=2.0 -parsedatetime>=2.1 +parsedatetime>=2.1,<2.6 pytz>=2015.4 mock>=1.3.0 isodate>=0.5.4 From 03c20e1faea859e4d3b80b9c68e435a8e9784d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Tue, 8 Sep 2020 02:03:56 +0200 Subject: [PATCH 006/163] Make code more legible by using agate.data_types.base.DEFAULT_NULL_VALUES --- agate/type_tester.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/agate/type_tester.py b/agate/type_tester.py index f1dd13b7..062428a0 100644 --- a/agate/type_tester.py +++ b/agate/type_tester.py @@ -2,6 +2,7 @@ from copy import copy +from agate.data_types.base import DEFAULT_NULL_VALUES from agate.data_types.boolean import Boolean from agate.data_types.date import Date from agate.data_types.date_time import DateTime @@ -53,16 +54,16 @@ class TypeTester(object): :class:`.Text`. Take care in specifying the order of the list. It is the order they are tested in. :class:`.Text` should always be last. :param null_values: - If :code:`types` is :code:`None`, a sequence of values which should be cast to - :code:`None` when encountered by the default data types. + If :code:`types` is :code:`None`, a sequence of values which should be + cast to :code:`None` when encountered by the default data types. """ - def __init__(self, force={}, limit=None, types=None, null_values=None): + def __init__(self, force={}, limit=None, types=None, null_values=DEFAULT_NULL_VALUES): self._force = force self._limit = limit if types: self._possible_types = types - elif null_values: + else: # In order of preference self._possible_types = [ Boolean(null_values=null_values), @@ -72,16 +73,6 @@ def __init__(self, force={}, limit=None, types=None, null_values=None): DateTime(null_values=null_values), Text(null_values=null_values) ] - else: - # In order of preference - self._possible_types = [ - Boolean(), - Number(), - TimeDelta(), - Date(), - DateTime(), - Text() - ] def run(self, rows, column_names): """ From 15296f14c0cf53ab8eb399064c439665221b134f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Tue, 15 Sep 2020 22:11:23 +0200 Subject: [PATCH 007/163] Avoid an unexpected AttributeError when pyicu.DateFormat.createTimeInstance() returns None (no idea why it does sometimes: buggy dll?) Workaround: in case of failure, avoid PyICU use in parsedatetime Traceback below: column_types = TypeTester(force=column_types) File "/usr/local/lib/python3.8/site-packages/agate/type_tester.py", line 68, in __init__ Date(), File "/usr/local/lib/python3.8/site-packages/agate/data_types/date.py", line 34, in __init__ self._constants = parsedatetime.Constants(localeID=self.locale, usePyICU=True) File "/usr/local/lib/python3.8/site-packages/parsedatetime/__init__.py", line 2381, in __init__ self.locale = get_icu(self.localeID) File "/usr/local/lib/python3.8/site-packages/parsedatetime/pdt_locales/icu.py", line 100, in get_icu result['timeFormats'][x] = icu_tf[x].toPattern() AttributeError: 'NoneType' object has no attribute 'toPattern' --- agate/data_types/date.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/agate/data_types/date.py b/agate/data_types/date.py index 57e329b6..9f15cfe4 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -31,7 +31,10 @@ def __init__(self, date_format=None, locale=None, **kwargs): self.date_format = date_format self.locale = locale - self._constants = parsedatetime.Constants(localeID=self.locale, usePyICU=True) + try: + self._constants = parsedatetime.Constants(localeID=self.locale, usePyICU=True) + except AttributeError: + self._constants = parsedatetime.Constants(localeID=self.locale, usePyICU=False) self._parser = parsedatetime.Calendar(constants=self._constants, version=parsedatetime.VERSION_CONTEXT_STYLE) def __getstate__(self): From 191929db6a8214368130def9c212aa30036e4134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Tue, 15 Sep 2020 22:39:05 +0200 Subject: [PATCH 008/163] Fix end-of-line spacing --- agate/type_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agate/type_tester.py b/agate/type_tester.py index 062428a0..338c9c3f 100644 --- a/agate/type_tester.py +++ b/agate/type_tester.py @@ -54,7 +54,7 @@ class TypeTester(object): :class:`.Text`. Take care in specifying the order of the list. It is the order they are tested in. :class:`.Text` should always be last. :param null_values: - If :code:`types` is :code:`None`, a sequence of values which should be + If :code:`types` is :code:`None`, a sequence of values which should be cast to :code:`None` when encountered by the default data types. """ def __init__(self, force={}, limit=None, types=None, null_values=DEFAULT_NULL_VALUES): From e72d554503579641652dc5368b9f9bdb5557075e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Tue, 15 Sep 2020 22:39:32 +0200 Subject: [PATCH 009/163] Revert "Avoid an unexpected AttributeError when pyicu.DateFormat.createTimeInstance() returns None" This reverts commit 15296f14c0cf53ab8eb399064c439665221b134f. --- agate/data_types/date.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/agate/data_types/date.py b/agate/data_types/date.py index 9f15cfe4..57e329b6 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -31,10 +31,7 @@ def __init__(self, date_format=None, locale=None, **kwargs): self.date_format = date_format self.locale = locale - try: - self._constants = parsedatetime.Constants(localeID=self.locale, usePyICU=True) - except AttributeError: - self._constants = parsedatetime.Constants(localeID=self.locale, usePyICU=False) + self._constants = parsedatetime.Constants(localeID=self.locale, usePyICU=True) self._parser = parsedatetime.Calendar(constants=self._constants, version=parsedatetime.VERSION_CONTEXT_STYLE) def __getstate__(self): From c1e312e5963d4949fee3aa3d8b553585e291c821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Wed, 16 Sep 2020 13:43:33 +0200 Subject: [PATCH 010/163] Warn instead of raising a ValueError when some keys in the 'force' dict do not match the table's column_names --- agate/type_tester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agate/type_tester.py b/agate/type_tester.py index e153fdb3..0d70c160 100644 --- a/agate/type_tester.py +++ b/agate/type_tester.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from copy import copy +import warnings from agate.data_types.boolean import Boolean from agate.data_types.date import Date @@ -86,7 +87,7 @@ def run(self, rows, column_names): try: force_indices.append(column_names.index(name)) except ValueError: - raise ValueError('"%s" does not match the name of any column in this table.' % name) + warnings.warn('"%s" does not match the name of any column in this table.' % name, RuntimeWarning) if self._limit: sample_rows = rows[:self._limit] From d52ea16d8b8877b2bd92a40f1528406c2efdfa46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20CORBASSON?= Date: Tue, 8 Sep 2020 01:16:51 +0200 Subject: [PATCH 011/163] Pin parsedatetime to <2.6 while waiting for upstream fixes --- requirements-py2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 9bc3a8d8..6a10474e 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -8,7 +8,7 @@ sphinx_rtd_theme>=0.1.6 wheel>=0.24.0 pytimeparse>=1.1.5 Babel>=2.0 -parsedatetime>=2.1 +parsedatetime>=2.1,<2.6 pytz>=2015.4 mock>=1.3.0 isodate>=0.5.4 From 5ba7ab3fb66fb0fbb802a3db46d7ab0b2352a869 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Thu, 31 Dec 2020 07:43:03 +1100 Subject: [PATCH 012/163] docs: fix simple typo, peform -> perform There is a small typo in agate/table/join.py. Should read `perform` rather than `peform`. --- agate/table/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agate/table/join.py b/agate/table/join.py index 48ee5cac..1f9bc54d 100644 --- a/agate/table/join.py +++ b/agate/table/join.py @@ -11,7 +11,7 @@ def join(self, right_table, left_key=None, right_key=None, inner=False, full_out implements most varieties of SQL join, in addition to some unique features. If :code:`left_key` and :code:`right_key` are both :code:`None` then this - method will peform a "sequential join", which is to say it will join on row + method will perform a "sequential join", which is to say it will join on row number. The :code:`inner` and :code:`full_outer` arguments will determine whether dangling left-hand and right-hand rows are included, respectively. From 4f52693fdddb2fe7dc07c75b84cb28731d2d5ae6 Mon Sep 17 00:00:00 2001 From: mathdesc Date: Mon, 22 Feb 2021 23:57:49 +0100 Subject: [PATCH 013/163] add support print_html with max_precision --- agate/table/print_html.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/agate/table/print_html.py b/agate/table/print_html.py index 411a6bd3..b5da68a5 100644 --- a/agate/table/print_html.py +++ b/agate/table/print_html.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # pylint: disable=W0212 +import math import sys from babel.numbers import format_decimal @@ -11,7 +12,7 @@ from agate import utils -def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_width=20, locale=None): +def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_width=20, locale=None, max_precision=3): """ Print an HTML version of this table. @@ -32,6 +33,10 @@ def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_w :param locale: Provide a locale you would like to be used to format the output. By default it will use the system's setting. + :max_precision: + Puts a limit on the maximum precision displayed for number types. + Numbers with lesser precision won't be affected. + This defaults to :code:`3`. Pass :code:`None` to disable limit. """ if max_rows is None: max_rows = len(self._rows) @@ -39,6 +44,9 @@ def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_w if max_columns is None: max_columns = len(self._columns) + if max_precision is None: + max_precision = float('inf') + ellipsis = config.get_option('ellipsis_chars') locale = locale or config.get_option('default_locale') @@ -60,7 +68,11 @@ def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_w if isinstance(c.data_type, Number): max_places = utils.max_precision(c[:max_rows]) - number_formatters.append(utils.make_number_formatter(max_places)) + add_ellipsis = False + if max_places > max_precision: + add_ellipsis = True + max_places = max_precision + number_formatters.append(utils.make_number_formatter(max_places, add_ellipsis)) else: number_formatters.append(None) @@ -76,7 +88,7 @@ def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_w v = ellipsis elif v is None: v = '' - elif number_formatters[j] is not None: + elif number_formatters[j] is not None and not math.isinf(v): v = format_decimal( v, format=number_formatters[j], From 86cdf2ace3a337a2233bc51e2fa835265b2b3d3a Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 10 Mar 2021 17:04:12 -0500 Subject: [PATCH 014/163] changelog: Set date --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index edc9aad9..7db20363 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,5 @@ -1.6.2 - Unreleased ------------------- +1.6.2 - March 10, 2021 +---------------------- * :meth:`.Date.__init__` and :meth:`.DateTime.__init__` accepts a ``locale`` keyword argument (e.g. :code:`en_US`) for parsing formatted dates. (#730) * :meth:`.utils.max_precision` ignores infinity when calculating precision. (#726) From f7b3297b8512163a06f86d5b94bed893b507a619 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 12 Jul 2021 18:17:11 -0400 Subject: [PATCH 015/163] build: Make PyICU an optional dependency --- .travis.yml | 6 ------ CHANGELOG.rst | 5 +++++ agate/data_types/date.py | 4 ++-- agate/data_types/date_time.py | 4 ++-- docs/install.rst | 2 ++ setup.py | 1 - 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74b429f3..baa7c701 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,12 +59,6 @@ jobs: - export LD_LIBRARY_PATH="$PATH:$PWD/vcpkg/installed/x64-windows/bin:$PWD/vcpkg/installed/x64-windows/lib" - export PYICU_INCLUDES="$PWD/vcpkg/installed/x64-windows/include" - export PYICU_LFLAGS='/LIBPATH:$PWD/vcpkg/installed/x64-windows/bin' -# - wget -q https://github.com/unicode-org/icu/releases/download/release-65-1/icu4c-65_1-Win64-MSVC2017.zip -O icu4c.zip -# - unzip -q icu4c.zip -d icu4c -# - export PATH="$PATH:$PWD/icu4c/bin64" -# - export LD_LIBRARY_PATH="$PATH:$PWD/icu4c/bin64:$PWD/icu4c/lib64" -# - export PYICU_INCLUDES="$PWD/icu4c/include" -# - export PYICU_LFLAGS='/LIBPATH:$PWD/icu4c/lib64' - uconv -V - export ICU_VERSION="$(uconv -V | sed -e 's,.*\`__. + .. note:: Need more speed? Upgrade to Python 3. It's 3-5x faster than Python 2. diff --git a/setup.py b/setup.py index f797dbba..be9ef7ce 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ 'isodate>=0.5.4', 'python-slugify>=1.2.1', 'leather>=0.3.2', - 'PyICU>=2.4.2', ] setup( From a9f77b3e5fd9fcaf119a168ab5435374b25381f4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 12 Jul 2021 18:29:40 -0400 Subject: [PATCH 016/163] build: Exclude bad version of parsedatetime, closes #743 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index be9ef7ce..14e5cd0f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ install_requires = [ 'six>=1.9.0', 'pytimeparse>=1.1.5', - 'parsedatetime>=2.1', + 'parsedatetime>=2.1,!=2.5', 'Babel>=2.0', 'isodate>=0.5.4', 'python-slugify>=1.2.1', From 9fb39a9d142f810f7b49428d363f58c7a17d7f1c Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 17:29:28 -0400 Subject: [PATCH 017/163] build: Set release to version --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 826f7101..1f17df4a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ project = u'agate' copyright = u'2017, Christopher Groskopf' version = '1.6.2' -release = '1.6.2' +release = version exclude_patterns = ['_build'] pygments_style = 'sphinx' From c3f05f04a90b289f2dfcfbe8e4f4abb399f00952 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 17:32:11 -0400 Subject: [PATCH 018/163] chore: Run isort --- agate/__init__.py | 5 +++-- agate/aggregations/__init__.py | 9 ++++----- agate/aggregations/stdev.py | 2 +- agate/computations/__init__.py | 5 ++--- agate/computations/percent_change.py | 1 - agate/computations/slug.py | 2 +- agate/config.py | 1 - agate/data_types/__init__.py | 2 +- agate/data_types/boolean.py | 2 +- agate/data_types/date.py | 3 +-- agate/data_types/date_time.py | 2 +- agate/data_types/number.py | 2 +- agate/fixed.py | 1 - agate/mapped_sequence.py | 1 + agate/table/__init__.py | 6 +++--- agate/table/bins.py | 2 +- agate/table/denormalize.py | 4 ++-- agate/table/from_csv.py | 1 + agate/table/from_fixed.py | 3 +-- agate/table/from_json.py | 5 +++-- agate/table/homogenize.py | 2 +- agate/table/join.py | 2 +- agate/table/normalize.py | 4 ++-- agate/table/pivot.py | 2 +- agate/table/print_bars.py | 8 +++----- agate/table/print_html.py | 5 ++--- agate/table/print_table.py | 5 ++--- agate/table/select.py | 2 +- agate/table/to_json.py | 2 +- agate/tableset/__init__.py | 5 ++--- agate/tableset/from_csv.py | 2 +- agate/tableset/from_json.py | 4 ++-- agate/tableset/to_json.py | 2 +- agate/utils.py | 9 ++++++--- charts.py | 1 - exonerations.py | 3 ++- tests/test_aggregations.py | 2 +- tests/test_computations.py | 4 ++-- tests/test_data_types.py | 3 ++- tests/test_fixed.py | 3 +-- tests/test_from_json.py | 5 +++-- tests/test_py3.py | 3 ++- tests/test_table/__init__.py | 2 +- tests/test_table/test_bins.py | 1 + tests/test_table/test_compute.py | 2 +- tests/test_table/test_denormalize.py | 2 +- tests/test_table/test_from_csv.py | 2 +- tests/test_table/test_homogenize.py | 1 + tests/test_table/test_merge.py | 2 +- tests/test_table/test_normalize.py | 2 +- tests/test_table/test_order_py.py | 2 +- tests/test_table/test_print_bars.py | 4 ++-- tests/test_table/test_print_table.py | 2 +- tests/test_table/test_rename.py | 2 +- tests/test_table/test_to_csv.py | 6 +++--- tests/test_table/test_to_json.py | 5 +++-- tests/test_tableset/__init__.py | 4 ++-- tests/test_utils.py | 2 +- 58 files changed, 89 insertions(+), 89 deletions(-) diff --git a/agate/__init__.py b/agate/__init__.py index cac0342f..7542bad1 100644 --- a/agate/__init__.py +++ b/agate/__init__.py @@ -3,10 +3,10 @@ import six from agate.aggregations import * -from agate.data_types import * from agate.columns import Column # noqa from agate.computations import * from agate.config import get_option, set_option, set_options # noqa +from agate.data_types import * from agate.exceptions import * # import agate.fixed as fixed # noqa from agate.mapped_sequence import MappedSequence # noqa @@ -16,7 +16,8 @@ from agate.testcase import AgateTestCase # noqa from agate.type_tester import TypeTester # noqa from agate.utils import * -from agate.warns import NullCalculationWarning, DuplicateColumnWarning, warn_null_calculation, warn_duplicate_column # noqa +from agate.warns import (DuplicateColumnWarning, NullCalculationWarning, warn_duplicate_column, # noqa + warn_null_calculation) if six.PY2: # pragma: no cover import agate.csv_py2 as csv # noqa diff --git a/agate/aggregations/__init__.py b/agate/aggregations/__init__.py index cf82a30f..f12b3195 100644 --- a/agate/aggregations/__init__.py +++ b/agate/aggregations/__init__.py @@ -15,19 +15,18 @@ with a column for each aggregation and a row for each table in the set. """ -from agate.aggregations.base import Aggregation # noqa - from agate.aggregations.all import All # noqa from agate.aggregations.any import Any # noqa +from agate.aggregations.base import Aggregation # noqa from agate.aggregations.count import Count # noqa from agate.aggregations.deciles import Deciles # noqa from agate.aggregations.first import First # noqa from agate.aggregations.has_nulls import HasNulls # noqa from agate.aggregations.iqr import IQR # noqa from agate.aggregations.mad import MAD # noqa +from agate.aggregations.max import Max # noqa from agate.aggregations.max_length import MaxLength # noqa from agate.aggregations.max_precision import MaxPrecision # noqa -from agate.aggregations.max import Max # noqa from agate.aggregations.mean import Mean # noqa from agate.aggregations.median import Median # noqa from agate.aggregations.min import Min # noqa @@ -35,7 +34,7 @@ from agate.aggregations.percentiles import Percentiles # noqa from agate.aggregations.quartiles import Quartiles # noqa from agate.aggregations.quintiles import Quintiles # noqa -from agate.aggregations.stdev import StDev, PopulationStDev # noqa +from agate.aggregations.stdev import PopulationStDev, StDev # noqa from agate.aggregations.sum import Sum # noqa from agate.aggregations.summary import Summary # noqa -from agate.aggregations.variance import Variance, PopulationVariance # noqa +from agate.aggregations.variance import PopulationVariance, Variance # noqa diff --git a/agate/aggregations/stdev.py b/agate/aggregations/stdev.py index 74f18862..21f03ab5 100644 --- a/agate/aggregations/stdev.py +++ b/agate/aggregations/stdev.py @@ -2,7 +2,7 @@ from agate.aggregations import Aggregation from agate.aggregations.has_nulls import HasNulls -from agate.aggregations.variance import Variance, PopulationVariance +from agate.aggregations.variance import PopulationVariance, Variance from agate.data_types import Number from agate.exceptions import DataTypeError from agate.warns import warn_null_calculation diff --git a/agate/computations/__init__.py b/agate/computations/__init__.py index 14bdf52e..666f620d 100644 --- a/agate/computations/__init__.py +++ b/agate/computations/__init__.py @@ -14,11 +14,10 @@ class by inheriting from :class:`Computation`. """ from agate.computations.base import Computation # noqa - -from agate.computations.formula import Formula # noqa from agate.computations.change import Change # noqa +from agate.computations.formula import Formula # noqa from agate.computations.percent import Percent # noqa from agate.computations.percent_change import PercentChange # noqa -from agate.computations.rank import Rank # noqa from agate.computations.percentile_rank import PercentileRank # noqa +from agate.computations.rank import Rank # noqa from agate.computations.slug import Slug # noqa diff --git a/agate/computations/percent_change.py b/agate/computations/percent_change.py index 8c287941..3a2dd485 100644 --- a/agate/computations/percent_change.py +++ b/agate/computations/percent_change.py @@ -2,7 +2,6 @@ from agate.aggregations.has_nulls import HasNulls from agate.computations.base import Computation - from agate.data_types import Number from agate.exceptions import DataTypeError from agate.warns import warn_null_calculation diff --git a/agate/computations/slug.py b/agate/computations/slug.py index b6780e68..68a00fd4 100644 --- a/agate/computations/slug.py +++ b/agate/computations/slug.py @@ -4,7 +4,7 @@ from agate.computations.base import Computation from agate.data_types import Text from agate.exceptions import DataTypeError -from agate.utils import slugify, issequence +from agate.utils import issequence, slugify class Slug(Computation): diff --git a/agate/config.py b/agate/config.py index f3c11dbb..fc112ec4 100644 --- a/agate/config.py +++ b/agate/config.py @@ -34,7 +34,6 @@ from babel.core import default_locale - _options = { #: Default locale for number formatting 'default_locale': default_locale('LC_NUMERIC') or 'en_US', diff --git a/agate/data_types/__init__.py b/agate/data_types/__init__.py index 1a9cef43..73e61d5a 100644 --- a/agate/data_types/__init__.py +++ b/agate/data_types/__init__.py @@ -10,7 +10,7 @@ """ from agate.data_types.base import DEFAULT_NULL_VALUES, DataType # noqa -from agate.data_types.boolean import Boolean, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES # noqa +from agate.data_types.boolean import DEFAULT_FALSE_VALUES, DEFAULT_TRUE_VALUES, Boolean # noqa from agate.data_types.date import Date # noqa from agate.data_types.date_time import DateTime # noqa from agate.data_types.number import Number # noqa diff --git a/agate/data_types/boolean.py b/agate/data_types/boolean.py index ae890bd2..6b4908dc 100644 --- a/agate/data_types/boolean.py +++ b/agate/data_types/boolean.py @@ -7,7 +7,7 @@ import six -from agate.data_types.base import DataType, DEFAULT_NULL_VALUES +from agate.data_types.base import DEFAULT_NULL_VALUES, DataType from agate.exceptions import CastError #: Default values which will be automatically cast to :code:`True`. diff --git a/agate/data_types/date.py b/agate/data_types/date.py index debf42ac..926bacf0 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -1,16 +1,15 @@ #!/usr/bin/env python +import locale from datetime import date, datetime, time import isodate -import locale import parsedatetime import six from agate.data_types.base import DataType from agate.exceptions import CastError - ZERO_DT = datetime.combine(date.min, time.min) diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index f7a02348..84bc2b4e 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -1,9 +1,9 @@ #!/usr/bin/env python import datetime +import locale import isodate -import locale import parsedatetime import six diff --git a/agate/data_types/number.py b/agate/data_types/number.py index a92b95ec..86bd2513 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -8,8 +8,8 @@ import warnings -from babel.core import Locale import six +from babel.core import Locale from agate.data_types.base import DataType from agate.exceptions import CastError diff --git a/agate/fixed.py b/agate/fixed.py index 944e0126..66f4fad9 100644 --- a/agate/fixed.py +++ b/agate/fixed.py @@ -9,7 +9,6 @@ import six - Field = namedtuple('Field', ['name', 'start', 'length']) diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index 16fb3c92..6429c5f3 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -7,6 +7,7 @@ """ from collections import OrderedDict + try: from collections.abc import Sequence except ImportError: diff --git a/agate/table/__init__.py b/agate/table/__init__.py index 150d9ee9..a01e8246 100644 --- a/agate/table/__init__.py +++ b/agate/table/__init__.py @@ -19,20 +19,20 @@ rows, row names are optional.) """ -from itertools import chain import sys import warnings +from itertools import chain import six from six.moves import range # pylint: disable=W0622 +from agate import utils from agate.columns import Column from agate.data_types import DataType +from agate.exceptions import CastError from agate.mapped_sequence import MappedSequence from agate.rows import Row from agate.type_tester import TypeTester -from agate import utils -from agate.exceptions import CastError from agate.warns import warn_duplicate_column, warn_unnamed_column diff --git a/agate/table/bins.py b/agate/table/bins.py index b4f3fe2f..0acc9a83 100644 --- a/agate/table/bins.py +++ b/agate/table/bins.py @@ -8,8 +8,8 @@ from babel.numbers import format_decimal -from agate.aggregations import Min, Max from agate import utils +from agate.aggregations import Max, Min def bins(self, column_name, count=10, start=None, end=None): diff --git a/agate/table/denormalize.py b/agate/table/denormalize.py index beec713b..d1aa4231 100644 --- a/agate/table/denormalize.py +++ b/agate/table/denormalize.py @@ -10,10 +10,10 @@ import six +from agate import utils from agate.data_types import Number -from agate.type_tester import TypeTester from agate.rows import Row -from agate import utils +from agate.type_tester import TypeTester def denormalize(self, key=None, property_column='property', value_column='value', default_value=utils.default, column_types=None): diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index 1e962a97..d95cead0 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import io + import six diff --git a/agate/table/from_fixed.py b/agate/table/from_fixed.py index f73fee85..de88fd3b 100644 --- a/agate/table/from_fixed.py +++ b/agate/table/from_fixed.py @@ -2,8 +2,7 @@ import io -from agate import fixed -from agate import utils +from agate import fixed, utils @classmethod diff --git a/agate/table/from_json.py b/agate/table/from_json.py index 8516702a..823fb754 100644 --- a/agate/table/from_json.py +++ b/agate/table/from_json.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -from collections import OrderedDict -from decimal import Decimal import io import json +from collections import OrderedDict +from decimal import Decimal + import six diff --git a/agate/table/homogenize.py b/agate/table/homogenize.py index dd29a2d8..0e04ec4e 100644 --- a/agate/table/homogenize.py +++ b/agate/table/homogenize.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # pylint: disable=W0212 -from agate.rows import Row from agate import utils +from agate.rows import Row def homogenize(self, key, compare_values, default_row=None): diff --git a/agate/table/join.py b/agate/table/join.py index 48ee5cac..58aee034 100644 --- a/agate/table/join.py +++ b/agate/table/join.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # pylint: disable=W0212 -from agate.rows import Row from agate import utils +from agate.rows import Row def join(self, right_table, left_key=None, right_key=None, inner=False, full_outer=False, require_match=False, columns=None): diff --git a/agate/table/normalize.py b/agate/table/normalize.py index 3b941f59..7c5b9779 100644 --- a/agate/table/normalize.py +++ b/agate/table/normalize.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # pylint: disable=W0212 -from agate.type_tester import TypeTester -from agate.rows import Row from agate import utils +from agate.rows import Row +from agate.type_tester import TypeTester def normalize(self, key, properties, property_column='property', value_column='value', column_types=None): diff --git a/agate/table/pivot.py b/agate/table/pivot.py index 1d90cbc7..9db82aaf 100644 --- a/agate/table/pivot.py +++ b/agate/table/pivot.py @@ -3,8 +3,8 @@ import six -from agate.aggregations import Count from agate import utils +from agate.aggregations import Count def pivot(self, key=None, pivot=None, aggregation=None, computation=None, default_value=utils.default, key_name=None): diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index 838d42a4..7700dd12 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -11,15 +11,13 @@ import sys - -from babel.numbers import format_decimal import six +from babel.numbers import format_decimal -from agate.aggregations import Min, Max -from agate import config +from agate import config, utils +from agate.aggregations import Max, Min from agate.data_types import Number from agate.exceptions import DataTypeError -from agate import utils def print_bars(self, label_column_name='group', value_column_name='Count', domain=None, width=120, output=sys.stdout, printable=False): diff --git a/agate/table/print_html.py b/agate/table/print_html.py index 411a6bd3..7489e782 100644 --- a/agate/table/print_html.py +++ b/agate/table/print_html.py @@ -3,12 +3,11 @@ import sys -from babel.numbers import format_decimal import six +from babel.numbers import format_decimal -from agate import config +from agate import config, utils from agate.data_types import Number, Text -from agate import utils def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_width=20, locale=None): diff --git a/agate/table/print_table.py b/agate/table/print_table.py index 7158511b..228d6cce 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -4,12 +4,11 @@ import math import sys -from babel.numbers import format_decimal import six +from babel.numbers import format_decimal -from agate import config +from agate import config, utils from agate.data_types import Number, Text -from agate import utils def print_table(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_width=20, locale=None, max_precision=3): diff --git a/agate/table/select.py b/agate/table/select.py index 3321738a..a1cd1ff9 100644 --- a/agate/table/select.py +++ b/agate/table/select.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # pylint: disable=W0212 -from agate.rows import Row from agate import utils +from agate.rows import Row def select(self, key): diff --git a/agate/table/to_json.py b/agate/table/to_json.py index 55346e15..97ab4a8f 100644 --- a/agate/table/to_json.py +++ b/agate/table/to_json.py @@ -2,9 +2,9 @@ # pylint: disable=W0212 import codecs -from collections import OrderedDict import json import os +from collections import OrderedDict import six diff --git a/agate/tableset/__init__.py b/agate/tableset/__init__.py index cc5b0a55..de00c0e9 100644 --- a/agate/tableset/__init__.py +++ b/agate/tableset/__init__.py @@ -168,9 +168,8 @@ def _proxy(self, method_name, *args, **kwargs): from agate.tableset.line_chart import line_chart from agate.tableset.merge import merge from agate.tableset.print_structure import print_structure -from agate.tableset.proxy_methods import bins, compute, denormalize, distinct, \ - exclude, find, group_by, homogenize, join, limit, normalize, order_by, \ - pivot, select, where +from agate.tableset.proxy_methods import (bins, compute, denormalize, distinct, exclude, find, group_by, homogenize, + join, limit, normalize, order_by, pivot, select, where) from agate.tableset.scatterplot import scatterplot from agate.tableset.to_csv import to_csv from agate.tableset.to_json import to_json diff --git a/agate/tableset/from_csv.py b/agate/tableset/from_csv.py index 81af9b49..64b7e2e8 100644 --- a/agate/tableset/from_csv.py +++ b/agate/tableset/from_csv.py @@ -1,8 +1,8 @@ #!/usr/bin/env python +import os from collections import OrderedDict from glob import glob -import os from agate.table import Table diff --git a/agate/tableset/from_json.py b/agate/tableset/from_json.py index b2befe44..cd069120 100644 --- a/agate/tableset/from_json.py +++ b/agate/tableset/from_json.py @@ -1,10 +1,10 @@ #!/usr/bin/env python +import json +import os from collections import OrderedDict from decimal import Decimal from glob import glob -import json -import os import six diff --git a/agate/tableset/to_json.py b/agate/tableset/to_json.py index 2a0ca262..963bab8a 100644 --- a/agate/tableset/to_json.py +++ b/agate/tableset/to_json.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -from collections import OrderedDict import json import os +from collections import OrderedDict import six diff --git a/agate/utils.py b/agate/utils.py index ae6aab9f..60a3c6f2 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -7,25 +7,28 @@ """ from collections import OrderedDict + try: from collections.abc import Sequence except ImportError: from collections import Sequence -from functools import wraps + import math import string import warnings +from functools import wraps + from slugify import slugify as pslugify + from agate.warns import warn_duplicate_column, warn_unnamed_column try: - from cdecimal import Decimal, ROUND_FLOOR, ROUND_CEILING, getcontext + from cdecimal import ROUND_CEILING, ROUND_FLOOR, Decimal, getcontext except ImportError: # pragma: no cover from decimal import Decimal, ROUND_FLOOR, ROUND_CEILING, getcontext import six - #: Sentinal for use when `None` is an valid argument value default = object() diff --git a/charts.py b/charts.py index d228c9ee..53f1abf2 100644 --- a/charts.py +++ b/charts.py @@ -2,7 +2,6 @@ import agate - table = agate.Table.from_csv('examples/realdata/Datagov_FY10_EDU_recp_by_State.csv') table.limit(10).bar_chart('State Name', 'TOTAL', 'docs/images/bar_chart.svg') diff --git a/exonerations.py b/exonerations.py index 923600d1..5baf80c0 100755 --- a/exonerations.py +++ b/exonerations.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -import agate import proof +import agate + def load_data(data): data['exonerations'] = agate.Table.from_csv('examples/realdata/exonerations-20150828.csv') diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 3722b5f3..fda4d457 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -2,10 +2,10 @@ # -*- coding: utf8 -*- import datetime -from decimal import Decimal import platform import sys import warnings +from decimal import Decimal try: import unittest2 as unittest diff --git a/tests/test_computations.py b/tests/test_computations.py index b8c118ae..c40a7262 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -1,8 +1,8 @@ #!/usr/bin/env Python import datetime -from decimal import Decimal import warnings +from decimal import Decimal try: import unittest2 as unittest @@ -10,8 +10,8 @@ import unittest from agate import Table -from agate.data_types import * from agate.computations import * +from agate.data_types import * from agate.exceptions import * from agate.warns import NullCalculationWarning diff --git a/tests/test_data_types.py b/tests/test_data_types.py index 4476631d..fddd798a 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -2,8 +2,9 @@ # -*- coding: utf8 -*- import datetime -from decimal import Decimal import pickle +from decimal import Decimal + import parsedatetime try: diff --git a/tests/test_fixed.py b/tests/test_fixed.py index a8305610..5ce42b56 100644 --- a/tests/test_fixed.py +++ b/tests/test_fixed.py @@ -5,8 +5,7 @@ except ImportError: import unittest -from agate import csv -from agate import fixed +from agate import csv, fixed class TestFixed(unittest.TestCase): diff --git a/tests/test_from_json.py b/tests/test_from_json.py index 3674711d..d4fbaf99 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -1,11 +1,12 @@ #!/usr/bin/env python # -*- coding: utf8 -*- +import six + from agate import Table -from agate.testcase import AgateTestCase from agate.data_types import * +from agate.testcase import AgateTestCase from agate.type_tester import TypeTester -import six class TestJSON(AgateTestCase): diff --git a/tests/test_py3.py b/tests/test_py3.py index 1a85d9c0..29c68a96 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -2,9 +2,10 @@ # -*- coding: utf-8 -*- import csv -import six import os +import six + try: import unittest2 as unittest except ImportError: diff --git a/tests/test_table/__init__.py b/tests/test_table/__init__.py index f16aee42..2607d9b6 100644 --- a/tests/test_table/__init__.py +++ b/tests/test_table/__init__.py @@ -12,8 +12,8 @@ import six from agate import Table -from agate.data_types import * from agate.computations import Formula +from agate.data_types import * from agate.testcase import AgateTestCase from agate.warns import DuplicateColumnWarning diff --git a/tests/test_table/test_bins.py b/tests/test_table/test_bins.py index 4245d3ee..883164b7 100644 --- a/tests/test_table/test_bins.py +++ b/tests/test_table/test_bins.py @@ -2,6 +2,7 @@ # -*- coding: utf8 -*- from babel.numbers import get_decimal_symbol + try: from cdecimal import Decimal except ImportError: # pragma: no cover diff --git a/tests/test_table/test_compute.py b/tests/test_table/test_compute.py index 62589fe1..fe3b2878 100644 --- a/tests/test_table/test_compute.py +++ b/tests/test_table/test_compute.py @@ -4,8 +4,8 @@ import six from agate import Table -from agate.data_types import * from agate.computations import Formula +from agate.data_types import * from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_denormalize.py b/tests/test_table/test_denormalize.py index cedf8e15..267b9ab1 100644 --- a/tests/test_table/test_denormalize.py +++ b/tests/test_table/test_denormalize.py @@ -3,8 +3,8 @@ from agate import Table from agate.data_types import * -from agate.type_tester import TypeTester from agate.testcase import AgateTestCase +from agate.type_tester import TypeTester class TestDenormalize(AgateTestCase): diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index 560813d8..772397ea 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -7,8 +7,8 @@ import six from agate import Table -from agate.testcase import AgateTestCase from agate.data_types import * +from agate.testcase import AgateTestCase from agate.type_tester import TypeTester diff --git a/tests/test_table/test_homogenize.py b/tests/test_table/test_homogenize.py index 9c9ad7b3..3d5fda87 100644 --- a/tests/test_table/test_homogenize.py +++ b/tests/test_table/test_homogenize.py @@ -2,6 +2,7 @@ # -*- coding: utf8 -*- from six.moves import range + from agate import Table from agate.data_types import * from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_merge.py b/tests/test_table/test_merge.py index 2e2fff23..62f023bf 100644 --- a/tests/test_table/test_merge.py +++ b/tests/test_table/test_merge.py @@ -3,8 +3,8 @@ from agate import Table from agate.data_types import * -from agate.testcase import AgateTestCase from agate.exceptions import DataTypeError +from agate.testcase import AgateTestCase class TestMerge(AgateTestCase): diff --git a/tests/test_table/test_normalize.py b/tests/test_table/test_normalize.py index 68f2f742..e5afb0f6 100644 --- a/tests/test_table/test_normalize.py +++ b/tests/test_table/test_normalize.py @@ -3,8 +3,8 @@ from agate import Table from agate.data_types import * -from agate.type_tester import TypeTester from agate.testcase import AgateTestCase +from agate.type_tester import TypeTester class TestNormalize(AgateTestCase): diff --git a/tests/test_table/test_order_py.py b/tests/test_table/test_order_py.py index 4f0d1c8d..757df907 100644 --- a/tests/test_table/test_order_py.py +++ b/tests/test_table/test_order_py.py @@ -2,8 +2,8 @@ # -*- coding: utf8 -*- from agate import Table -from agate.testcase import AgateTestCase from agate.data_types import * +from agate.testcase import AgateTestCase class TestOrderBy(AgateTestCase): diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index 5026280f..1e4b3558 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -from babel.numbers import format_decimal import six +from babel.numbers import format_decimal from agate import Table from agate.data_types import * -from agate.testcase import AgateTestCase from agate.exceptions import DataTypeError +from agate.testcase import AgateTestCase class TestPrintBars(AgateTestCase): diff --git a/tests/test_table/test_print_table.py b/tests/test_table/test_print_table.py index 7efc3079..ae44ff7a 100644 --- a/tests/test_table/test_print_table.py +++ b/tests/test_table/test_print_table.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -from babel.numbers import get_decimal_symbol import six +from babel.numbers import get_decimal_symbol from agate import Table from agate.data_types import * diff --git a/tests/test_table/test_rename.py b/tests/test_table/test_rename.py index 859d91ea..936fffd4 100644 --- a/tests/test_table/test_rename.py +++ b/tests/test_table/test_rename.py @@ -4,8 +4,8 @@ import warnings from agate import Table -from agate.testcase import AgateTestCase from agate.data_types import * +from agate.testcase import AgateTestCase class TestRename(AgateTestCase): diff --git a/tests/test_table/test_to_csv.py b/tests/test_table/test_to_csv.py index 9939bf49..11b1836c 100644 --- a/tests/test_table/test_to_csv.py +++ b/tests/test_table/test_to_csv.py @@ -1,14 +1,14 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import six - import os import sys +import six + from agate import Table -from agate.testcase import AgateTestCase from agate.data_types import * +from agate.testcase import AgateTestCase class TestToCSV(AgateTestCase): diff --git a/tests/test_table/test_to_json.py b/tests/test_table/test_to_json.py index 376afc09..42c7bb11 100644 --- a/tests/test_table/test_to_json.py +++ b/tests/test_table/test_to_json.py @@ -1,14 +1,15 @@ #!/usr/bin/env python # -*- coding: utf8 -*- +import json import os import sys + import six -import json from agate import Table -from agate.testcase import AgateTestCase from agate.data_types import * +from agate.testcase import AgateTestCase class TestJSON(AgateTestCase): diff --git a/tests/test_tableset/__init__.py b/tests/test_tableset/__init__.py index 8fc38b0f..0ca9a5d3 100644 --- a/tests/test_tableset/__init__.py +++ b/tests/test_tableset/__init__.py @@ -7,13 +7,13 @@ except ImportError: from io import StringIO -import shutil import json +import shutil from agate import Table, TableSet from agate.aggregations import * -from agate.data_types import * from agate.computations import Formula +from agate.data_types import * from agate.testcase import AgateTestCase diff --git a/tests/test_utils.py b/tests/test_utils.py index 3705cc4d..0fb971ab 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,7 +16,7 @@ from agate.data_types import Text from agate.mapped_sequence import MappedSequence from agate.table import Table -from agate.utils import Quantiles, round_limits, letter_name +from agate.utils import Quantiles, letter_name, round_limits class TestQuantiles(unittest.TestCase): From 6d3fdf16012e00cdcae7374504aab24d1e6a56f3 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 17:55:17 -0400 Subject: [PATCH 019/163] chore: flake8 tests --- tests/test_aggregations.py | 11 +++++------ tests/test_columns.py | 2 +- tests/test_computations.py | 6 +++--- tests/test_data_types.py | 7 ++++--- tests/test_from_json.py | 2 +- tests/test_mapped_sequence.py | 1 - tests/test_table/__init__.py | 6 +++--- tests/test_table/test_bins.py | 2 +- tests/test_table/test_charting.py | 5 ----- tests/test_table/test_compute.py | 2 +- tests/test_table/test_denormalize.py | 2 +- tests/test_table/test_from_csv.py | 3 +-- tests/test_table/test_group_by.py | 2 +- tests/test_table/test_homogenize.py | 2 +- tests/test_table/test_join.py | 2 +- tests/test_table/test_merge.py | 2 +- tests/test_table/test_normalize.py | 2 +- tests/test_table/test_order_py.py | 2 +- tests/test_table/test_pivot.py | 2 +- tests/test_table/test_print_bars.py | 6 +++--- tests/test_table/test_print_structure.py | 2 +- tests/test_table/test_print_table.py | 2 +- tests/test_table/test_rename.py | 2 +- tests/test_table/test_to_csv.py | 2 +- tests/test_table/test_to_json.py | 2 +- tests/test_tableset/__init__.py | 3 +-- tests/test_tableset/test_aggregate.py | 4 ++-- tests/test_tableset/test_charting.py | 5 ----- tests/test_tableset/test_merge.py | 3 +-- tests/test_type_tester.py | 2 +- tests/test_utils.py | 6 ------ 31 files changed, 41 insertions(+), 61 deletions(-) diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index fda4d457..3a924418 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -2,7 +2,6 @@ # -*- coding: utf8 -*- import datetime -import platform import sys import warnings from decimal import Decimal @@ -12,12 +11,12 @@ except ImportError: import unittest -import six - from agate import Table -from agate.aggregations import * -from agate.data_types import * -from agate.exceptions import * +from agate.aggregations import (All, Any, Count, Deciles, First, HasNulls, IQR, MAD, Max, MaxLength, MaxPrecision, + Mean, Median, Min, Mode, Percentiles, PopulationStDev, PopulationVariance, Quartiles, + Quintiles, StDev, Sum, Summary, Variance) +from agate.data_types import Boolean, Number, DateTime, Text, TimeDelta +from agate.exceptions import DataTypeError from agate.warns import NullCalculationWarning diff --git a/tests/test_columns.py b/tests/test_columns.py index 22ec9d79..107219e7 100644 --- a/tests/test_columns.py +++ b/tests/test_columns.py @@ -14,7 +14,7 @@ import unittest from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text class TestColumn(unittest.TestCase): diff --git a/tests/test_computations.py b/tests/test_computations.py index c40a7262..c94482a1 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -10,9 +10,9 @@ import unittest from agate import Table -from agate.computations import * -from agate.data_types import * -from agate.exceptions import * +from agate.computations import Change, Formula, Percent, PercentChange, PercentileRank, Rank, Slug +from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta +from agate.exceptions import CastError, DataTypeError from agate.warns import NullCalculationWarning diff --git a/tests/test_data_types.py b/tests/test_data_types.py index fddd798a..0281b806 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -6,6 +6,7 @@ from decimal import Decimal import parsedatetime +import six try: import unittest2 as unittest @@ -15,7 +16,7 @@ import pytz from agate.columns import * -from agate.data_types import * +from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.exceptions import CastError @@ -154,8 +155,8 @@ def test_cast(self): @unittest.skipIf(six.PY3, 'Not supported in Python 3.') def test_cast_long(self): - self.assertEqual(self.type.test(long('141414')), True) - self.assertEqual(self.type.cast(long('141414')), Decimal('141414')) + self.assertEqual(self.type.test(long('141414')), True) # noqa: F405 + self.assertEqual(self.type.cast(long('141414')), Decimal('141414')) # noqa: F405 def test_boolean_cast(self): values = (True, False) diff --git a/tests/test_from_json.py b/tests/test_from_json.py index d4fbaf99..5dfd4fb5 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -4,7 +4,7 @@ import six from agate import Table -from agate.data_types import * +from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.testcase import AgateTestCase from agate.type_tester import TypeTester diff --git a/tests/test_mapped_sequence.py b/tests/test_mapped_sequence.py index 6258d31a..b113ecc6 100644 --- a/tests/test_mapped_sequence.py +++ b/tests/test_mapped_sequence.py @@ -7,7 +7,6 @@ import six -from agate.data_types import * from agate.mapped_sequence import MappedSequence diff --git a/tests/test_table/__init__.py b/tests/test_table/__init__.py index 2607d9b6..d60a0d27 100644 --- a/tests/test_table/__init__.py +++ b/tests/test_table/__init__.py @@ -6,14 +6,14 @@ except ImportError: # pragma: no cover from decimal import Decimal -import platform import warnings import six from agate import Table from agate.computations import Formula -from agate.data_types import * +from agate.data_types import Number, Text +from agate.exceptions import CastError from agate.testcase import AgateTestCase from agate.warns import DuplicateColumnWarning @@ -304,7 +304,7 @@ def test_stringify(self): table = Table(self.rows, column_names) if six.PY2: - u = unicode(table) + u = unicode(table) # noqa: F821 self.assertIn('foo', u) self.assertIn('bar', u) diff --git a/tests/test_table/test_bins.py b/tests/test_table/test_bins.py index 883164b7..96795630 100644 --- a/tests/test_table/test_bins.py +++ b/tests/test_table/test_bins.py @@ -9,7 +9,7 @@ from decimal import Decimal from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_charting.py b/tests/test_table/test_charting.py index 9238dba1..1a543384 100644 --- a/tests/test_table/test_charting.py +++ b/tests/test_table/test_charting.py @@ -1,11 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -try: - import unittest2 as unittest -except ImportError: - import unittest - import leather from agate import Table diff --git a/tests/test_table/test_compute.py b/tests/test_table/test_compute.py index fe3b2878..e0bbca88 100644 --- a/tests/test_table/test_compute.py +++ b/tests/test_table/test_compute.py @@ -5,7 +5,7 @@ from agate import Table from agate.computations import Formula -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_denormalize.py b/tests/test_table/test_denormalize.py index 267b9ab1..9eb52598 100644 --- a/tests/test_table/test_denormalize.py +++ b/tests/test_table/test_denormalize.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase from agate.type_tester import TypeTester diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index 772397ea..28e3c6ba 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -7,7 +7,7 @@ import six from agate import Table -from agate.data_types import * +from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.testcase import AgateTestCase from agate.type_tester import TypeTester @@ -105,7 +105,6 @@ def test_from_csv_no_header_columns(self): self.assertColumnTypes(table, [Number, Text, Boolean, Date, DateTime, TimeDelta]) def test_from_csv_sniff_limit_0(self): - table1 = Table(self.rows, self.column_names, self.column_types) table2 = Table.from_csv('examples/test_csv_sniff.csv', sniff_limit=0) self.assertColumnNames(table2, ['number|text|boolean|date|datetime|timedelta']) diff --git a/tests/test_table/test_group_by.py b/tests/test_table/test_group_by.py index d3c8f95e..73751b2a 100644 --- a/tests/test_table/test_group_by.py +++ b/tests/test_table/test_group_by.py @@ -7,7 +7,7 @@ from decimal import Decimal from agate import Table, TableSet -from agate.data_types import * +from agate.data_types import Boolean, Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_homogenize.py b/tests/test_table/test_homogenize.py index 3d5fda87..ecb87a0e 100644 --- a/tests/test_table/test_homogenize.py +++ b/tests/test_table/test_homogenize.py @@ -4,7 +4,7 @@ from six.moves import range from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_join.py b/tests/test_table/test_join.py index 1201e06b..f4991c65 100644 --- a/tests/test_table/test_join.py +++ b/tests/test_table/test_join.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_merge.py b/tests/test_table/test_merge.py index 62f023bf..626b9a18 100644 --- a/tests/test_table/test_merge.py +++ b/tests/test_table/test_merge.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.exceptions import DataTypeError from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_normalize.py b/tests/test_table/test_normalize.py index e5afb0f6..9d131843 100644 --- a/tests/test_table/test_normalize.py +++ b/tests/test_table/test_normalize.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase from agate.type_tester import TypeTester diff --git a/tests/test_table/test_order_py.py b/tests/test_table/test_order_py.py index 757df907..6a3cd23b 100644 --- a/tests/test_table/test_order_py.py +++ b/tests/test_table/test_order_py.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_pivot.py b/tests/test_table/test_pivot.py index 68c51521..7ac90526 100644 --- a/tests/test_table/test_pivot.py +++ b/tests/test_table/test_pivot.py @@ -11,7 +11,7 @@ from agate import Table from agate.aggregations import Sum from agate.computations import Percent -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index 1e4b3558..0c31d503 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -5,7 +5,7 @@ from babel.numbers import format_decimal from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.exceptions import DataTypeError from agate.testcase import AgateTestCase @@ -43,7 +43,7 @@ def test_print_bars_width(self): table.print_bars('three', 'one', width=40, output=output) lines = output.getvalue().split('\n') - self.assertEqual(max([len(l) for l in lines]), 40) + self.assertEqual(max([len(line) for line in lines]), 40) def test_print_bars_width_overlap(self): table = Table(self.rows, self.column_names, self.column_types) @@ -52,7 +52,7 @@ def test_print_bars_width_overlap(self): table.print_bars('three', 'one', width=20, output=output) lines = output.getvalue().split('\n') - self.assertEqual(max([len(l) for l in lines]), 20) + self.assertEqual(max([len(line) for line in lines]), 20) def test_print_bars_domain(self): table = Table(self.rows, self.column_names, self.column_types) diff --git a/tests/test_table/test_print_structure.py b/tests/test_table/test_print_structure.py index 75f4ba90..18b47adf 100644 --- a/tests/test_table/test_print_structure.py +++ b/tests/test_table/test_print_structure.py @@ -4,7 +4,7 @@ import six from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_print_table.py b/tests/test_table/test_print_table.py index ae44ff7a..1d4aef76 100644 --- a/tests/test_table/test_print_table.py +++ b/tests/test_table/test_print_table.py @@ -5,7 +5,7 @@ from babel.numbers import get_decimal_symbol from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_rename.py b/tests/test_table/test_rename.py index 936fffd4..9ea5fd4e 100644 --- a/tests/test_table/test_rename.py +++ b/tests/test_table/test_rename.py @@ -4,7 +4,7 @@ import warnings from agate import Table -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_to_csv.py b/tests/test_table/test_to_csv.py index 11b1836c..917d40f9 100644 --- a/tests/test_table/test_to_csv.py +++ b/tests/test_table/test_to_csv.py @@ -7,7 +7,7 @@ import six from agate import Table -from agate.data_types import * +from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_to_json.py b/tests/test_table/test_to_json.py index 42c7bb11..d98eb36e 100644 --- a/tests/test_table/test_to_json.py +++ b/tests/test_table/test_to_json.py @@ -8,7 +8,7 @@ import six from agate import Table -from agate.data_types import * +from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.testcase import AgateTestCase diff --git a/tests/test_tableset/__init__.py b/tests/test_tableset/__init__.py index 0ca9a5d3..b4b1616c 100644 --- a/tests/test_tableset/__init__.py +++ b/tests/test_tableset/__init__.py @@ -11,9 +11,8 @@ import shutil from agate import Table, TableSet -from agate.aggregations import * from agate.computations import Formula -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_tableset/test_aggregate.py b/tests/test_tableset/test_aggregate.py index 19c252b8..676dda7a 100644 --- a/tests/test_tableset/test_aggregate.py +++ b/tests/test_tableset/test_aggregate.py @@ -8,8 +8,8 @@ from decimal import Decimal from agate import Table, TableSet -from agate.aggregations import * -from agate.data_types import * +from agate.aggregations import Count, Sum, MaxLength, Min, Mean +from agate.data_types import Number, Text from agate.exceptions import DataTypeError from agate.testcase import AgateTestCase diff --git a/tests/test_tableset/test_charting.py b/tests/test_tableset/test_charting.py index ad03c8bd..e90bf227 100644 --- a/tests/test_tableset/test_charting.py +++ b/tests/test_tableset/test_charting.py @@ -3,11 +3,6 @@ from collections import OrderedDict -try: - import unittest2 as unittest -except ImportError: - import unittest - import leather from agate import Table, TableSet diff --git a/tests/test_tableset/test_merge.py b/tests/test_tableset/test_merge.py index 9b3d21c7..6c3077db 100644 --- a/tests/test_tableset/test_merge.py +++ b/tests/test_tableset/test_merge.py @@ -3,8 +3,7 @@ from collections import OrderedDict from agate import Table, TableSet -from agate.aggregations import * -from agate.data_types import * +from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 6288f9d7..e8d801ff 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -6,7 +6,7 @@ except ImportError: import unittest -from agate.data_types import * +from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.type_tester import TypeTester diff --git a/tests/test_utils.py b/tests/test_utils.py index 0fb971ab..9df54fcb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,12 +10,6 @@ except ImportError: import unittest -import sys -import warnings - -from agate.data_types import Text -from agate.mapped_sequence import MappedSequence -from agate.table import Table from agate.utils import Quantiles, letter_name, round_limits From 9a57802e92a536ee53b58634be75059dbb09041f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:15:11 -0400 Subject: [PATCH 020/163] chore: Update authors --- AUTHORS.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 28091202..0f79cdbe 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -37,8 +37,10 @@ agate is made by a community. The following individuals have contributed code, d * `Neil MartinsenBurrell `_ * `Aliaksei Urbanski `_ * `Forest Gregg `_ +* `Robert Schütz `_ * `Wouter de Vries `_ * `Kartik Agaram `_ * `Loïc Corbasson `_ -* `Robert Schütz `_ * `Danny Sepler `_ +* `brian-from-quantrocket `_ + From 38ae2e374153ccb04ddf5193dc6bca3575570022 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:16:09 -0400 Subject: [PATCH 021/163] chore: flake8 agate --- agate/__init__.py | 24 ++++++++++++------------ agate/aggregations/all.py | 2 +- agate/aggregations/any.py | 2 +- agate/aggregations/first.py | 1 - agate/csv_py2.py | 6 ++++-- agate/csv_py3.py | 2 +- agate/data_types/date.py | 3 +-- agate/data_types/date_time.py | 6 +++--- agate/data_types/number.py | 2 +- agate/table/__init__.py | 1 - agate/table/bar_chart.py | 2 -- agate/table/column_chart.py | 2 -- agate/table/from_json.py | 2 -- agate/table/line_chart.py | 2 -- agate/table/print_bars.py | 4 ++-- agate/table/scatterplot.py | 2 -- agate/table/to_json.py | 2 +- agate/tableset/__init__.py | 1 - agate/tableset/bar_chart.py | 2 -- agate/tableset/column_chart.py | 2 -- agate/tableset/line_chart.py | 2 -- agate/tableset/proxy_methods.py | 14 ++++++++++++++ agate/tableset/scatterplot.py | 2 -- agate/utils.py | 1 - exonerations.py | 1 + tests/test_table/test_print_html.py | 1 + 26 files changed, 43 insertions(+), 48 deletions(-) diff --git a/agate/__init__.py b/agate/__init__.py index 7542bad1..3a98c7a8 100644 --- a/agate/__init__.py +++ b/agate/__init__.py @@ -3,23 +3,23 @@ import six from agate.aggregations import * -from agate.columns import Column # noqa +from agate.columns import Column from agate.computations import * -from agate.config import get_option, set_option, set_options # noqa +from agate.config import get_option, set_option, set_options from agate.data_types import * from agate.exceptions import * -# import agate.fixed as fixed # noqa -from agate.mapped_sequence import MappedSequence # noqa -from agate.rows import Row # noqa -from agate.table import Table # noqa -from agate.tableset import TableSet # noqa -from agate.testcase import AgateTestCase # noqa -from agate.type_tester import TypeTester # noqa +# import agate.fixed as fixed +from agate.mapped_sequence import MappedSequence +from agate.rows import Row +from agate.table import Table +from agate.tableset import TableSet +from agate.testcase import AgateTestCase +from agate.type_tester import TypeTester from agate.utils import * -from agate.warns import (DuplicateColumnWarning, NullCalculationWarning, warn_duplicate_column, # noqa +from agate.warns import (DuplicateColumnWarning, NullCalculationWarning, warn_duplicate_column, warn_null_calculation) if six.PY2: # pragma: no cover - import agate.csv_py2 as csv # noqa + import agate.csv_py2 as csv else: - import agate.csv_py3 as csv # noqa + import agate.csv_py3 as csv diff --git a/agate/aggregations/all.py b/agate/aggregations/all.py index 2a7f9290..f4ecc149 100644 --- a/agate/aggregations/all.py +++ b/agate/aggregations/all.py @@ -27,7 +27,7 @@ def get_aggregate_data_type(self, table): return Boolean() def validate(self, table): - column = table.columns[self._column_name] + table.columns[self._column_name] def run(self, table): """ diff --git a/agate/aggregations/any.py b/agate/aggregations/any.py index 8cea92a7..30ddd7e9 100644 --- a/agate/aggregations/any.py +++ b/agate/aggregations/any.py @@ -27,7 +27,7 @@ def get_aggregate_data_type(self, table): return Boolean() def validate(self, table): - column = table.columns[self._column_name] + table.columns[self._column_name] def run(self, table): column = table.columns[self._column_name] diff --git a/agate/aggregations/first.py b/agate/aggregations/first.py index 37e16950..287bfa13 100644 --- a/agate/aggregations/first.py +++ b/agate/aggregations/first.py @@ -1,7 +1,6 @@ #!/usr/bin/env python from agate.aggregations.base import Aggregation -from agate.data_types import Boolean class First(Aggregation): diff --git a/agate/csv_py2.py b/agate/csv_py2.py index 034c5877..32a113b5 100644 --- a/agate/csv_py2.py +++ b/agate/csv_py2.py @@ -228,7 +228,9 @@ def writerow(self, row): self._append_line_number(row) # Convert embedded Mac line endings to unix style line endings so they get quoted - row = dict([(k, v.replace('\r', '\n')) if isinstance(v, basestring) else (k, v) for k, v in row.items()]) + row = dict([ + (k, v.replace('\r', '\n')) if isinstance(v, basestring) else (k, v) for k, v in row.items() # noqa: F821 + ]) UnicodeDictWriter.writerow(self, row) @@ -248,7 +250,7 @@ def sniff(self, sample): """ try: dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except: + except Exception: dialect = None return dialect diff --git a/agate/csv_py3.py b/agate/csv_py3.py index 66f5b925..918af60a 100644 --- a/agate/csv_py3.py +++ b/agate/csv_py3.py @@ -151,7 +151,7 @@ def sniff(self, sample): """ try: dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except: + except Exception: dialect = None return dialect diff --git a/agate/data_types/date.py b/agate/data_types/date.py index 926bacf0..20bdb6e4 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -3,7 +3,6 @@ import locale from datetime import date, datetime, time -import isodate import parsedatetime import six @@ -81,7 +80,7 @@ def cast(self, d): try: dt = datetime.strptime(d, self.date_format) - except: + except (ValueError, TypeError): raise CastError('Value "%s" does not match date format.' % d) finally: if orig_locale: diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index 84bc2b4e..8cb9660a 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -89,7 +89,7 @@ def cast(self, d): try: dt = datetime.datetime.strptime(d, self.datetime_format) - except: + except (ValueError, TypeError): raise CastError('Value "%s" does not match date format.' % d) finally: if orig_locale: @@ -99,7 +99,7 @@ def cast(self, d): try: (_, _, _, _, matched_text), = self._parser.nlp(d, sourceTime=self._source_time) - except: + except Exception: matched_text = None else: value, ctx = self._parser.parseDT( @@ -117,7 +117,7 @@ def cast(self, d): dt = isodate.parse_datetime(d) return dt - except: + except Exception: pass raise CastError('Can not parse value "%s" as datetime.' % d) diff --git a/agate/data_types/number.py b/agate/data_types/number.py index 86bd2513..56bd3c31 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -66,7 +66,7 @@ def cast(self, d): if t is int: return Decimal(d) - elif six.PY2 and t is long: + elif six.PY2 and t is long: # noqa: F821 return Decimal(d) elif t is float: return Decimal(repr(d)) diff --git a/agate/table/__init__.py b/agate/table/__init__.py index a01e8246..fba388dc 100644 --- a/agate/table/__init__.py +++ b/agate/table/__init__.py @@ -33,7 +33,6 @@ from agate.mapped_sequence import MappedSequence from agate.rows import Row from agate.type_tester import TypeTester -from agate.warns import warn_duplicate_column, warn_unnamed_column @six.python_2_unicode_compatible diff --git a/agate/table/bar_chart.py b/agate/table/bar_chart.py index 9e3da515..2853dda1 100644 --- a/agate/table/bar_chart.py +++ b/agate/table/bar_chart.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def bar_chart(self, label=0, value=1, path=None, width=None, height=None): """ diff --git a/agate/table/column_chart.py b/agate/table/column_chart.py index 11bca44a..5ea31cf1 100644 --- a/agate/table/column_chart.py +++ b/agate/table/column_chart.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def column_chart(self, label=0, value=1, path=None, width=None, height=None): """ diff --git a/agate/table/from_json.py b/agate/table/from_json.py index 823fb754..78461ef6 100644 --- a/agate/table/from_json.py +++ b/agate/table/from_json.py @@ -5,8 +5,6 @@ from collections import OrderedDict from decimal import Decimal -import six - @classmethod def from_json(cls, path, row_names=None, key=None, newline=False, column_types=None, encoding='utf-8', **kwargs): diff --git a/agate/table/line_chart.py b/agate/table/line_chart.py index 6e4c680c..72c5f9df 100644 --- a/agate/table/line_chart.py +++ b/agate/table/line_chart.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def line_chart(self, x=0, y=1, path=None, width=None, height=None): """ diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index 7700dd12..cddbccc3 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -88,8 +88,8 @@ def print_bars(self, label_column_name='group', value_column_name='Count', domai locale=locale )) - max_label_width = max(max([len(l) for l in formatted_labels]), len(y_label)) - max_value_width = max(max([len(v) for v in formatted_values]), len(x_label)) + max_label_width = max(max([len(label) for label in formatted_labels]), len(y_label)) + max_value_width = max(max([len(value) for value in formatted_values]), len(x_label)) plot_width = width - (max_label_width + max_value_width + 2) diff --git a/agate/table/scatterplot.py b/agate/table/scatterplot.py index 0de29665..671c9723 100644 --- a/agate/table/scatterplot.py +++ b/agate/table/scatterplot.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def scatterplot(self, x=0, y=1, path=None, width=None, height=None): """ diff --git a/agate/table/to_json.py b/agate/table/to_json.py index 97ab4a8f..01333bd6 100644 --- a/agate/table/to_json.py +++ b/agate/table/to_json.py @@ -78,7 +78,7 @@ def dump_json(data): if key_is_row_function: k = key(row) else: - k = str(row[key]) if six.PY3 else unicode(row[key]) + k = str(row[key]) if six.PY3 else unicode(row[key]) # noqa: F821 if k in output: raise ValueError('Value %s is not unique in the key column.' % six.text_type(k)) diff --git a/agate/tableset/__init__.py b/agate/tableset/__init__.py index de00c0e9..5dbe58bf 100644 --- a/agate/tableset/__init__.py +++ b/agate/tableset/__init__.py @@ -29,7 +29,6 @@ from agate.data_types import Text from agate.mapped_sequence import MappedSequence -from agate.table import Table class TableSet(MappedSequence): diff --git a/agate/tableset/bar_chart.py b/agate/tableset/bar_chart.py index 4fd26b95..26041987 100644 --- a/agate/tableset/bar_chart.py +++ b/agate/tableset/bar_chart.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def bar_chart(self, label=0, value=1, path=None, width=None, height=None): """ diff --git a/agate/tableset/column_chart.py b/agate/tableset/column_chart.py index 4e5caf49..7116c488 100644 --- a/agate/tableset/column_chart.py +++ b/agate/tableset/column_chart.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def column_chart(self, label=0, value=1, path=None, width=None, height=None): """ diff --git a/agate/tableset/line_chart.py b/agate/tableset/line_chart.py index 77400886..e28aefb7 100644 --- a/agate/tableset/line_chart.py +++ b/agate/tableset/line_chart.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def line_chart(self, x=0, y=1, path=None, width=None, height=None): """ diff --git a/agate/tableset/proxy_methods.py b/agate/tableset/proxy_methods.py index e8657b62..b8207cde 100644 --- a/agate/tableset/proxy_methods.py +++ b/agate/tableset/proxy_methods.py @@ -7,84 +7,98 @@ def bins(self, *args, **kwargs): """ return self._proxy('bins', *args, **kwargs) + def compute(self, *args, **kwargs): """ Calls :meth:`.Table.compute` on each table in the TableSet. """ return self._proxy('compute', *args, **kwargs) + def denormalize(self, *args, **kwargs): """ Calls :meth:`.Table.denormalize` on each table in the TableSet. """ return self._proxy('denormalize', *args, **kwargs) + def distinct(self, *args, **kwargs): """ Calls :meth:`.Table.distinct` on each table in the TableSet. """ return self._proxy('distinct', *args, **kwargs) + def exclude(self, *args, **kwargs): """ Calls :meth:`.Table.exclude` on each table in the TableSet. """ return self._proxy('exclude', *args, **kwargs) + def find(self, *args, **kwargs): """ Calls :meth:`.Table.find` on each table in the TableSet. """ return self._proxy('find', *args, **kwargs) + def group_by(self, *args, **kwargs): """ Calls :meth:`.Table.group_by` on each table in the TableSet. """ return self._proxy('group_by', *args, **kwargs) + def homogenize(self, *args, **kwargs): """ Calls :meth:`.Table.homogenize` on each table in the TableSet. """ return self._proxy('homogenize', *args, **kwargs) + def join(self, *args, **kwargs): """ Calls :meth:`.Table.join` on each table in the TableSet. """ return self._proxy('join', *args, **kwargs) + def limit(self, *args, **kwargs): """ Calls :meth:`.Table.limit` on each table in the TableSet. """ return self._proxy('limit', *args, **kwargs) + def normalize(self, *args, **kwargs): """ Calls :meth:`.Table.normalize` on each table in the TableSet. """ return self._proxy('normalize', *args, **kwargs) + def order_by(self, *args, **kwargs): """ Calls :meth:`.Table.order_by` on each table in the TableSet. """ return self._proxy('order_by', *args, **kwargs) + def pivot(self, *args, **kwargs): """ Calls :meth:`.Table.pivot` on each table in the TableSet. """ return self._proxy('pivot', *args, **kwargs) + def select(self, *args, **kwargs): """ Calls :meth:`.Table.select` on each table in the TableSet. """ return self._proxy('select', *args, **kwargs) + def where(self, *args, **kwargs): """ Calls :meth:`.Table.where` on each table in the TableSet. diff --git a/agate/tableset/scatterplot.py b/agate/tableset/scatterplot.py index 0d9f554a..391280b4 100644 --- a/agate/tableset/scatterplot.py +++ b/agate/tableset/scatterplot.py @@ -3,8 +3,6 @@ import leather -from agate import utils - def scatterplot(self, x=0, y=1, path=None, width=None, height=None): """ diff --git a/agate/utils.py b/agate/utils.py index 60a3c6f2..1ce98175 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -15,7 +15,6 @@ import math import string -import warnings from functools import wraps from slugify import slugify as pslugify diff --git a/exonerations.py b/exonerations.py index 5baf80c0..9a7ba05d 100755 --- a/exonerations.py +++ b/exonerations.py @@ -89,6 +89,7 @@ def race_and_age(data): # Print out the results sorted_groups.print_table(max_rows=10) + analysis = proof.Analysis(load_data) analysis.then(confessions) analysis.then(median_age) diff --git a/tests/test_table/test_print_html.py b/tests/test_table/test_print_html.py index 7884b7cd..90d47daf 100644 --- a/tests/test_table/test_print_html.py +++ b/tests/test_table/test_print_html.py @@ -15,6 +15,7 @@ class TableHTMLParser(html_parser.HTMLParser): """ Parser for use in testing HTML rendering of tables. """ + def __init__(self): warnings.simplefilter('ignore') From 797c19f023ce551d0ffc8254e8cbf5c16d51f0cd Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:17:48 -0400 Subject: [PATCH 022/163] chore: flake8 agate/table/print_table.py --- agate/table/print_table.py | 1 - 1 file changed, 1 deletion(-) diff --git a/agate/table/print_table.py b/agate/table/print_table.py index 228d6cce..f3fa4712 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -142,7 +142,6 @@ def write_row(formatted_row): # Dashes span each width with '+' character at intersection of # horizontal and vertical dividers. divider = '%(v_line)s %(columns)s %(v_line)s' % { - 'h_line': h_line, 'v_line': v_line, 'columns': ' | '.join(h_line * w for w in widths) } From d0b1913ef0a74aaea3b51368d1828b16776a11a4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:18:23 -0400 Subject: [PATCH 023/163] chore: Add flake8 and isort configuration --- setup.cfg | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/setup.cfg b/setup.cfg index 2a9acf13..377c82f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,11 @@ +[flake8] +max-line-length = 119 +per-file-ignores = + # imported but unused + agate/__init__.py: F401 + +[isort] +line_length = 119 + [bdist_wheel] universal = 1 From c16ed62825a3b5ba9c538b899616cfce2fe18693 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:29:06 -0400 Subject: [PATCH 024/163] build: Move requirements into extras_require --- requirements-py2.txt | 19 --------------- requirements-py3.txt | 17 -------------- setup.py | 56 ++++++++++++++++++++++++-------------------- 3 files changed, 31 insertions(+), 61 deletions(-) delete mode 100644 requirements-py2.txt delete mode 100644 requirements-py3.txt diff --git a/requirements-py2.txt b/requirements-py2.txt deleted file mode 100644 index 9bc3a8d8..00000000 --- a/requirements-py2.txt +++ /dev/null @@ -1,19 +0,0 @@ -unittest2>=1.1.0 -nose>=1.1.2 -tox>=1.3 -Sphinx>=1.2.2 -coverage>=3.7.1 -six>=1.9.0 -sphinx_rtd_theme>=0.1.6 -wheel>=0.24.0 -pytimeparse>=1.1.5 -Babel>=2.0 -parsedatetime>=2.1 -pytz>=2015.4 -mock>=1.3.0 -isodate>=0.5.4 -python-slugify>=1.2.1 -lxml>=3.6.0,<4.0.0 -cssselect>=0.9.1 -leather>=0.3.2 -PyICU>=2.4.2 diff --git a/requirements-py3.txt b/requirements-py3.txt deleted file mode 100644 index c94c368b..00000000 --- a/requirements-py3.txt +++ /dev/null @@ -1,17 +0,0 @@ -nose>=1.1.2 -tox>=1.3 -Sphinx>=1.2.2 -coverage>=3.7.1 -six>=1.9.0 -sphinx_rtd_theme>=0.1.6 -wheel>=0.24.0 -pytimeparse>=1.1.5 -Babel>=2.0 -parsedatetime>=2.1 -pytz>=2015.4 -isodate>=0.5.4 -python-slugify>=1.2.1 -lxml>=3.6.0 -cssselect>=0.9.1 -leather>=0.3.2 -PyICU>=2.4.2 diff --git a/setup.py b/setup.py index 14e5cd0f..17b73d2a 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,14 @@ -#!/usr/bin/env python +from setuptools import find_packages, setup -from setuptools import setup - -install_requires = [ - 'six>=1.9.0', - 'pytimeparse>=1.1.5', - 'parsedatetime>=2.1,!=2.5', - 'Babel>=2.0', - 'isodate>=0.5.4', - 'python-slugify>=1.2.1', - 'leather>=0.3.2', -] +with open('README.rst') as f: + long_description = f.read() setup( name='agate', version='1.6.2', description='A data analysis library that is optimized for humans instead of machines.', - long_description=open('README.rst').read(), + long_description=long_description, + long_description_content_type='text/x-rst', author='Christopher Groskopf', author_email='chrisgroskopf@gmail.com', url='http://agate.readthedocs.org/', @@ -30,25 +22,39 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: Software Development :: Libraries :: Python Modules', ], - packages=[ - 'agate', - 'agate.aggregations', - 'agate.computations', - 'agate.data_types', - 'agate.table', - 'agate.tableset' + packages=find_packages(exclude=['benchmarks', 'tests', 'tests.*']), + install_requires=[ + 'Babel>=2.0', + 'isodate>=0.5.4', + 'leather>=0.3.2', + 'parsedatetime>=2.1,!=2.5', + 'python-slugify>=1.2.1', + 'pytimeparse>=1.1.5', + 'six>=1.9.0', ], - install_requires=install_requires + extras_require={ + 'test': [ + 'coverage>=3.7.1', + 'cssselect>=0.9.1', + 'lxml>=3.6.0,<4.0.0', + 'nose>=1.1.2', + 'PyICU>=2.4.2', + 'pytz>=2015.4', + 'mock>=1.3.0;python_version<"3"', + 'unittest2>=1.1.0;python_version<"3"', + ], + 'docs': [ + 'Sphinx>=1.2.2', + 'sphinx_rtd_theme>=0.1.6', + ], + } ) From 9e96e4b707ceef02ef24954a072a26fff6372745 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:31:36 -0400 Subject: [PATCH 025/163] ci: Switch to GitHub Actions --- .github/workflows/ci.yml | 29 +++++++++ .travis.yml | 127 --------------------------------------- 2 files changed, 29 insertions(+), 127 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..8fabf455 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI +on: [push, pull_request] +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + python-version: [3.6, 3.7, 3.8, 3.9, pypy-3.6, pypy-3.7] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + # https://github.com/actions/cache/blob/main/examples.md#using-a-script-to-get-cache-location + - id: pip-cache + run: python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" + - uses: actions/cache@v1 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - run: pip install --upgrade check-manifest flake8 isort setuptools + - run: check-manifest + - run: flake8 . + - run: isort . --check-only + - run: pip install .[test] + - run: nosetests --with-coverage --cover-package=agate diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index baa7c701..00000000 --- a/.travis.yml +++ /dev/null @@ -1,127 +0,0 @@ -language: python -os: linux -python: - - "3.8" - - "3.7" - - "3.6" - - "3.5" - - "2.7" - - "pypy3" - - "pypy3.5-6.0" - - "pypy3.5-7.0" - - "pypy3.6-7.0.0" - - "pypy" - - "pypy2.7-6.0" - - "pypy2.7-7.0.0" -jobs: - include: - - os: osx - python: "3.7" - osx_image: xcode11.2 # Python 3.7.4 running on macOS 10.14.4 - language: shell # 'language: python' is an error on Travis CI macOS - before_install: - - brew install pkg-config - - brew install icu4c - - export PATH="$PATH:/usr/local/opt/icu4c/bin" - - export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig" - - which uconv - - uconv -V - - export ICU_VERSION="$(uconv -V | sed -e 's,.*\ - if [[ "$TRAVIS_PYTHON_VERSION" == "2"* ]] || [[ "$TRAVIS_PYTHON_VERSION" == "pypy"* ]] && [[ "$TRAVIS_PYTHON_VERSION" != "pypy3"* ]]; then - pip install -r requirements-py2.txt; - else - pip3 install -r requirements-py3.txt; - fi -# command to run tests -script: - # pypy2 and pypy3 segfault on Travis CI if running all tests in the same process - - > - if [[ "$TRAVIS_PYTHON_VERSION" == "pypy" ]]; then - nosetests --collect-only -v tests 2>&1 \ - | grep -e 'ok$' \ - | while read func class etc; do - class="${class//[()]/}"; - class="${class%.*}:${class##*.}"; - nosetests -v "$class.$func"; - done || ( echo "$s" >> "script-failures.log" ); - if [ -e "script-failures.log" ]; then - exit 1; - fi; - elif [[ "$TRAVIS_PYTHON_VERSION" == "pypy3" ]]; then - find tests -type f -name "*.py" | while read s; do - ( [ ! -x "$s" ] && nosetests --no-byte-compile -s -v "$s" ) || ( echo "$s" >> "script-failures.log" ); - done; - if [ -e "script-failures.log" ]; then - exit 1; - fi; - else - nosetests --no-byte-compile --with-coverage tests; - fi -after_failure: - - > - if [ -e "script-failures.log" ]; then - echo $(cat "script-failures.log"); - fi -addons: - apt: - packages: - - language-pack-fr - - language-pack-de - - language-pack-ko - - pkg-config From 1f8b40af80227c283a95591fe199a1b24da39dff Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:44:18 -0400 Subject: [PATCH 026/163] chore: Merge .coveragerc into setup.cfg --- .coveragerc | 5 ----- setup.cfg | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 303d855a..00000000 --- a/.coveragerc +++ /dev/null @@ -1,5 +0,0 @@ -[run] -include = - agate/* -omit = - agate/csv_py2.py diff --git a/setup.cfg b/setup.cfg index 377c82f7..a364ea4e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,3 +9,9 @@ line_length = 119 [bdist_wheel] universal = 1 + +[coverage:run] +include = + agate/* +omit = + agate/csv_py2.py From 7458e5ca6f134d7567b6c9f8fc99407f32834979 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:44:49 -0400 Subject: [PATCH 027/163] chore: Remove tox configuration --- .github/CONTRIBUTING.md | 2 +- .gitignore | 1 - docs/contributing.rst | 1 - docs/install.rst | 1 - docs/release_process.rst | 2 +- tox.ini | 33 --------------------------------- 6 files changed, 2 insertions(+), 38 deletions(-) delete mode 100644 tox.ini diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 2e69c6ca..fcf365db 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -22,7 +22,7 @@ Contributors should use the following roadmap to guide them through the process 1. Fork the project on [GitHub]. 2. Check out the [issue tracker] and find a task that needs to be done and is of a scope you can realistically expect to complete in a few days. Don’t worry about the priority of the issues at first, but try to choose something you’ll enjoy. You’re much more likely to finish something to the point it can be merged if it’s something you really enjoy hacking on. 3. Comment on the ticket letting everyone know you’re going to be hacking on it so that nobody duplicates your effort. It’s also good practice to provide some general idea of how you plan on resolving the issue so that other developers can make suggestions. -4. Write tests for the feature you’re building. Follow the format of the existing tests in the test directory to see how this works. You can run all the tests with the command `nosetests tests`. (Or `tox` to run across all supported versions of Python.) +4. Write tests for the feature you’re building. Follow the format of the existing tests in the test directory to see how this works. You can run all the tests with the command `nosetests tests`. 5. Write the code. Try to stay consistent with the style and organization of the existing codebase. A good patch won’t be refused for stylistic reasons, but large parts of it may be rewritten and nobody wants that. 6. As you are coding, periodically merge in work from the master branch and verify you haven’t broken anything by running the test suite. 7. Write documentation. Seriously. diff --git a/.gitignore b/.gitignore index 00314608..bdd410c9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ *.pyc *.swp *.swo -.tox *.egg-info docs/_build dist diff --git a/docs/contributing.rst b/docs/contributing.rst index 4352c600..e35cc04f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -31,7 +31,6 @@ Hacker? We'd love to have you hack with us. Please follow this process to make y #. If you don't have a specific task in mind, check out the `issue tracker `_ and find a task that needs to be done and is of a scope you can realistically expect to complete in a few days. Don't worry about the priority of the issues at first, but try to choose something you'll enjoy. You're much more likely to finish something to the point it can be merged if it's something you really enjoy hacking on. #. If you already have a task you know you want to work on, open a ticket or comment on the existing ticket letting everyone know you're going to be working on it. It's also good practice to provide some general idea of how you plan on resolving the issue so that other developers can make suggestions. #. Write tests for the feature you're building. Follow the format of the existing tests in the test directory to see how this works. You can run all the tests with the command ``nosetests tests``. -#. Verify your tests work on all supported versions of Python by runing ``tox``. #. Write the code. Try to stay consistent with the style and organization of the existing codebase. A good patch won't be refused for stylistic reasons, but large parts of it may be rewritten and nobody wants that. #. As you are coding, periodically merge in work from the master branch and verify you haven't broken anything by running the test suite. #. Write documentation. This means docstrings on all classes and methods, including parameter explanations. It also means, when relevant, cookbook recipes and updates to the agate user tutorial. diff --git a/docs/install.rst b/docs/install.rst index 237875d2..849ad10e 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -33,7 +33,6 @@ If you are a developer that also wants to hack on agate, install it from git:: pip install -r requirements-py2.txt python setup.py develop - tox .. note:: diff --git a/docs/release_process.rst b/docs/release_process.rst index 180a3a2c..3d22749c 100644 --- a/docs/release_process.rst +++ b/docs/release_process.rst @@ -4,7 +4,7 @@ Release process This is the release process for agate: -1. Verify all unit tests pass with fresh environments: ``tox -r``. +#. Verify all tests pass on continuous integration. 2. Verify 100% test coverage: ``nosetests --with-coverage tests``. 3. Ensure any new modules have been added to setup.py's ``packages`` list. #. Ensure any new public interfaces have been added to the documentation. diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 1fc719e9..00000000 --- a/tox.ini +++ /dev/null @@ -1,33 +0,0 @@ -[tox] -envlist = py27,py35,py36,py37,py38,pypy2,pypy3 - -[testenv] -commands=nosetests tests - -[testenv:py27] -deps = -rrequirements-py2.txt - -[testenv:py35] -deps = -rrequirements-py3.txt - -[testenv:py36] -deps = {[testenv:py35]deps} - -[testenv:py37] -deps = {[testenv:py35]deps} - -[testenv:py38] -deps = {[testenv:py35]deps} - -[testenv:pypy2] -deps = {[testenv:py27]deps} - -[testenv:pypy3] -deps = {[testenv:py35]deps} - -[flake8] -ignore=E128,E402,E501,F403 -# E128 continuation line under-indented for visual indent -# E402 module level import not at top of file -# E501 line too long (X > 79 characters) -# F403 'from xyz import *' used; unable to detect undefined names From d0c5b940cec30c9bb47fad689bc7c26015dfdf02 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:45:07 -0400 Subject: [PATCH 028/163] build: Update MANIFEST.in --- MANIFEST.in | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 835552a8..4a480615 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,13 @@ -include CHANGELOG.rst COPYING -recursive-include docs * -recursive-include tests * -recursive-include examples * +include *.ipynb +include *.py +include *.rst +include COPYING +recursive-include benchmarks *.py +recursive-include docs *.py +recursive-include docs *.rst +recursive-include docs *.svg +recursive-include docs Makefile +recursive-include examples *.csv +recursive-include examples *.json +recursive-include examples testfixed +recursive-include tests *.py From def485f40f0d32a2de00d03533d71e55e0eded01 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:50:26 -0400 Subject: [PATCH 029/163] chore: flake8 --- setup.cfg | 8 ++++++-- tests/test_data_types.py | 5 ++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index a364ea4e..e75805f2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,12 @@ [flake8] max-line-length = 119 +extend-ignore = E501 per-file-ignores = - # imported but unused - agate/__init__.py: F401 + # imported but unused, unable to detect undefined names + agate/__init__.py: F401,F403 + # module level import not at top of file + agate/tableset/__init__.py: E402 + agate/table/__init__.py: E402 [isort] line_length = 119 diff --git a/tests/test_data_types.py b/tests/test_data_types.py index 0281b806..e0d7f377 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -15,7 +15,6 @@ import pytz -from agate.columns import * from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.exceptions import CastError @@ -155,8 +154,8 @@ def test_cast(self): @unittest.skipIf(six.PY3, 'Not supported in Python 3.') def test_cast_long(self): - self.assertEqual(self.type.test(long('141414')), True) # noqa: F405 - self.assertEqual(self.type.cast(long('141414')), Decimal('141414')) # noqa: F405 + self.assertEqual(self.type.test(long('141414')), True) # noqa: F821 + self.assertEqual(self.type.cast(long('141414')), Decimal('141414')) # noqa: F821 def test_boolean_cast(self): values = (True, False) From f2db77b9d363f8afdaeac435a6b596dd63405bfe Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:58:16 -0400 Subject: [PATCH 030/163] chore: isort --- agate/__init__.py | 3 +-- tests/test_aggregations.py | 4 ++-- tests/test_tableset/test_aggregate.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/agate/__init__.py b/agate/__init__.py index 3a98c7a8..910eb7dc 100644 --- a/agate/__init__.py +++ b/agate/__init__.py @@ -16,8 +16,7 @@ from agate.testcase import AgateTestCase from agate.type_tester import TypeTester from agate.utils import * -from agate.warns import (DuplicateColumnWarning, NullCalculationWarning, warn_duplicate_column, - warn_null_calculation) +from agate.warns import DuplicateColumnWarning, NullCalculationWarning, warn_duplicate_column, warn_null_calculation if six.PY2: # pragma: no cover import agate.csv_py2 as csv diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 3a924418..09279afe 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -12,10 +12,10 @@ import unittest from agate import Table -from agate.aggregations import (All, Any, Count, Deciles, First, HasNulls, IQR, MAD, Max, MaxLength, MaxPrecision, +from agate.aggregations import (IQR, MAD, All, Any, Count, Deciles, First, HasNulls, Max, MaxLength, MaxPrecision, Mean, Median, Min, Mode, Percentiles, PopulationStDev, PopulationVariance, Quartiles, Quintiles, StDev, Sum, Summary, Variance) -from agate.data_types import Boolean, Number, DateTime, Text, TimeDelta +from agate.data_types import Boolean, DateTime, Number, Text, TimeDelta from agate.exceptions import DataTypeError from agate.warns import NullCalculationWarning diff --git a/tests/test_tableset/test_aggregate.py b/tests/test_tableset/test_aggregate.py index 676dda7a..b870d0d9 100644 --- a/tests/test_tableset/test_aggregate.py +++ b/tests/test_tableset/test_aggregate.py @@ -8,7 +8,7 @@ from decimal import Decimal from agate import Table, TableSet -from agate.aggregations import Count, Sum, MaxLength, Min, Mean +from agate.aggregations import Count, MaxLength, Mean, Min, Sum from agate.data_types import Number, Text from agate.exceptions import DataTypeError from agate.testcase import AgateTestCase From d8218f145763a04f2ef3f4a5f53b9c1d8fc10521 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 19:02:14 -0400 Subject: [PATCH 031/163] build: Only test PyICU on Linux --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 17b73d2a..6d25560b 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0,<4.0.0', 'nose>=1.1.2', - 'PyICU>=2.4.2', + 'PyICU>=2.4.2;sys_platform==linux', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', 'unittest2>=1.1.0;python_version<"3"', From 2f123c5a8b7940489934bd2e9ee9e1761c339341 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 19:04:12 -0400 Subject: [PATCH 032/163] build: Use correct quoting in platform-specific dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d25560b..edd78475 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0,<4.0.0', 'nose>=1.1.2', - 'PyICU>=2.4.2;sys_platform==linux', + 'PyICU>=2.4.2;sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', 'unittest2>=1.1.0;python_version<"3"', From 806b5d38bf82331e56e577e4e2e2a8039e9791a7 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 19:46:24 -0400 Subject: [PATCH 033/163] build: Unpin lxml in tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index edd78475..2c3ed024 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ 'test': [ 'coverage>=3.7.1', 'cssselect>=0.9.1', - 'lxml>=3.6.0,<4.0.0', + 'lxml>=3.6.0', 'nose>=1.1.2', 'PyICU>=2.4.2;sys_platform=="linux"', 'pytz>=2015.4', From 31f5f17eaf6a5f346158bc4a7c227b37a137d454 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:02:39 -0400 Subject: [PATCH 034/163] test: Fix import --- agate/tableset/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agate/tableset/merge.py b/agate/tableset/merge.py index aa1df5fc..a469fa64 100644 --- a/agate/tableset/merge.py +++ b/agate/tableset/merge.py @@ -2,7 +2,7 @@ # pylint: disable=W0212 from agate.rows import Row -from agate.tableset import Table +from agate.table import Table def merge(self, groups=None, group_name=None, group_type=None): From 091f9174912e29337bd9632934215d670a304ca0 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:19:39 -0400 Subject: [PATCH 035/163] test: Only test performance on Linux --- benchmarks/test_joins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index c6f57969..58098e16 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf8 -*- +import platform from random import shuffle from timeit import Timer @@ -15,6 +16,7 @@ import agate +@unittest.skipIf(platform.system() != "Linux") class TestTableJoin(unittest.TestCase): def test_join(self): left_rows = [(six.text_type(i), i) for i in range(100000)] From 60476d7031804a251445b4945c3fb5cbe4cddcf4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:24:43 -0400 Subject: [PATCH 036/163] ci: Move some release steps into continuous integration --- .github/workflows/ci.yml | 2 ++ docs/release_process.rst | 8 ++------ setup.cfg | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fabf455..973fa1ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,3 +27,5 @@ jobs: - run: isort . --check-only - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate + - run: python example.py + - run: python charts.py diff --git a/docs/release_process.rst b/docs/release_process.rst index 3d22749c..015d4c2e 100644 --- a/docs/release_process.rst +++ b/docs/release_process.rst @@ -5,19 +5,15 @@ Release process This is the release process for agate: #. Verify all tests pass on continuous integration. -2. Verify 100% test coverage: ``nosetests --with-coverage tests``. 3. Ensure any new modules have been added to setup.py's ``packages`` list. #. Ensure any new public interfaces have been added to the documentation. #. Ensure TableSet proxy methods have been added for new Table methods. -#. Make sure the example script still works: ``python example.py``. -#. Ensure ``python charts.py`` works and has been run recently. +#. Run ``python charts.py`` to update images in the documentation. #. Ensure ``CHANGELOG.rst`` is up to date. Add the release date and summary. #. Create a release tag: ``git tag -a x.y.z -m "x.y.z release."`` -#. Push tags upstream: ``git push --tags`` -#. If this is a major release, merge ``master`` into ``stable``: ``git checkout stable; git merge master; git push`` +#. Push tags upstream: ``git push --follow-tags`` #. Upload to `PyPI `_: ``python setup.py sdist bdist_wheel upload``. #. Flag the release to build on `RTFD `_. -#. Update the "default version" on `RTFD `_ to the latest. #. Rev to latest version: ``docs/conf.py``, ``setup.py``, ``CHANGELOG.rst`` need updates. #. Find/replace ``en/[old version]`` to ``en/[new version]`` in ``tutorial.ipynb``. #. Commit revision: ``git commit -am "Update to version x.y.z for development."``. diff --git a/setup.cfg b/setup.cfg index e75805f2..8b71880f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,5 @@ line_length = 119 universal = 1 [coverage:run] -include = - agate/* omit = agate/csv_py2.py From 2e096883b01d26ad7dfb2d8a34acdd75f8bdbf45 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:38:39 -0400 Subject: [PATCH 037/163] docs: Update changelog --- CHANGELOG.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e40cf8dc..95f58a14 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,12 @@ -Unreleased ----------- - -* Make PyICU an optional dependency. +1.6.3 - Unreleased +------------------ + +* feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) +* feat: :class:`.Max` works with :class:`.TimeDelta`. (#735) +* fix: :class:`.Mean` returns ``None`` if there are no values to average. (#706) +* fix: :meth:`.Table.homogenize` accepts tuples. (#710) +* fix: Ensure files are closed when errors occur. (#734) +* build: Make PyICU an optional dependency. 1.6.2 - March 10, 2021 ---------------------- From ba77654152f03a7cc5da602d63078f357798c8c3 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:40:17 -0400 Subject: [PATCH 038/163] docs: Remove a step from the release process --- docs/release_process.rst | 1 - tutorial.ipynb | 88 ++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/docs/release_process.rst b/docs/release_process.rst index 015d4c2e..0444b5e9 100644 --- a/docs/release_process.rst +++ b/docs/release_process.rst @@ -15,5 +15,4 @@ This is the release process for agate: #. Upload to `PyPI `_: ``python setup.py sdist bdist_wheel upload``. #. Flag the release to build on `RTFD `_. #. Rev to latest version: ``docs/conf.py``, ``setup.py``, ``CHANGELOG.rst`` need updates. -#. Find/replace ``en/[old version]`` to ``en/[new version]`` in ``tutorial.ipynb``. #. Commit revision: ``git commit -am "Update to version x.y.z for development."``. diff --git a/tutorial.ipynb b/tutorial.ipynb index aa0b20ec..89ee741e 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -23,7 +23,7 @@ "\n", "Note: You should be installing agate inside a [virtualenv](https://virtualenv.readthedocs.io/en/stable/>). If for some crazy reason you aren't using virtualenv you will need to add a ``sudo`` to the previous command.*\n", "\n", - "For more detailed installation instructions, see the [Installation](http://agate.readthedocs.io/en/1.6.2/install.html) section of the documentation." + "For more detailed installation instructions, see the [Installation](https://agate.readthedocs.io/en/latest/install.html) section of the documentation." ] }, { @@ -67,7 +67,7 @@ "source": [ "## Loading data from a CSV\n", "\n", - "The [`Table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#module-agate.table) is the basic class in agate. To create a table from a CSV we use the [`Table.from_csv`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.from_csv) class method:" + "The [`Table`](https://agate.readthedocs.io/en/latest/api/table.html#module-agate.table) is the basic class in agate. To create a table from a CSV we use the [`Table.from_csv`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.from_csv) class method:" ] }, { @@ -85,7 +85,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "With no other arguments specified, agate will automatically create an instance of [`TypeTester`](http://agate.readthedocs.io/en/1.6.2/api/type_tester.html#agate.TypeTester) and use it to figure out the type of each column. TypeTester is a \"best guess\" approach to determining the kinds of data in your table. It can guess wrong. In that case you can create a TypeTester manually and use the ``force`` argument to override its guess for a specific column:" + "With no other arguments specified, agate will automatically create an instance of [`TypeTester`](https://agate.readthedocs.io/en/latest/api/type_tester.html#agate.TypeTester) and use it to figure out the type of each column. TypeTester is a \"best guess\" approach to determining the kinds of data in your table. It can guess wrong. In that case you can create a TypeTester manually and use the ``force`` argument to override its guess for a specific column:" ] }, { @@ -107,9 +107,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If you already know the types of your data you may wish to skip the TypeTester entirely. You may pass sequences of column names and column types to [`Table.from_csv`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.from_csv) as the ``column_names`` and ``column_types`` arguments, respectively.\n", + "If you already know the types of your data you may wish to skip the TypeTester entirely. You may pass sequences of column names and column types to [`Table.from_csv`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.from_csv) as the ``column_names`` and ``column_types`` arguments, respectively.\n", "\n", - "For larger datasets the [`TypeTester`](http://agate.readthedocs.io/en/1.6.2/api/type_tester.html#agate.TypeTester) can be slow to evaluate the data. In that case you can specify a `limit` argument to restrict the amount of data it will use to infer types:" + "For larger datasets the [`TypeTester`](https://agate.readthedocs.io/en/latest/api/type_tester.html#agate.TypeTester) can be slow to evaluate the data. In that case you can specify a `limit` argument to restrict the amount of data it will use to infer types:" ] }, { @@ -133,7 +133,7 @@ "\n", "**Note:** agate's CSV reader and writer support unicode and other encodings for both Python 2 and Python 3. Try using them as a drop-in replacement for Python's builtin module: `from agate import csv`.\n", "\n", - "**Note:** agate also has [`Table.from_json`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.from_json) for creating tables from JSON data." + "**Note:** agate also has [`Table.from_json`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.from_json) for creating tables from JSON data." ] }, { @@ -143,7 +143,7 @@ "Describing the table\n", "====================\n", "\n", - "If you're working with new data, or you just need a refresher, you may want to review what columns are in the table. You can do this with the [`.Table.print_structure`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.print_structure) method or by just calling `print` on the table:" + "If you're working with new data, or you just need a refresher, you may want to review what columns are in the table. You can do this with the [`.Table.print_structure`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.print_structure) method or by just calling `print` on the table:" ] }, { @@ -192,9 +192,9 @@ "Navigating table data\n", "=====================\n", "\n", - "agate goes to great pains to make accessing the data in your tables work seamlessly for a wide variety of use-cases. Access by both [`Column`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.Column) and [`Row`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.Row) is supported, via the [`Table.columns`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.columns) and [`Table.rows`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.rows) attributes respectively.\n", + "agate goes to great pains to make accessing the data in your tables work seamlessly for a wide variety of use-cases. Access by both [`Column`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.Column) and [`Row`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.Row) is supported, via the [`Table.columns`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.columns) and [`Table.rows`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.rows) attributes respectively.\n", "\n", - "All four of these objects are examples of [`.MappedSequence`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.MappedSequence), the foundational type that underlies much of agate's functionality. A MappedSequence functions very similar to a standard Python [`dict`](https://docs.python.org/3/tutorial/datastructures.html#dictionaries), with a few important exceptions:\n", + "All four of these objects are examples of [`.MappedSequence`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.MappedSequence), the foundational type that underlies much of agate's functionality. A MappedSequence functions very similar to a standard Python [`dict`](https://docs.python.org/3/tutorial/datastructures.html#dictionaries), with a few important exceptions:\n", "\n", "* Data may be accessed either by numeric index (e.g. column number) or by a non-integer key (e.g. column name).\n", "* Items are ordered, just like an instance of [`collections.OrderedDict`](https://docs.python.org/3.5/library/collections.html#collections.OrderedDict).\n", @@ -412,7 +412,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For any instance of [`.MappedSequence`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.MappedSequence), iteration returns values, *in order*. Here we print only the first ten:" + "For any instance of [`.MappedSequence`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.MappedSequence), iteration returns values, *in order*. Here we print only the first ten:" ] }, { @@ -450,7 +450,7 @@ "collapsed": true }, "source": [ - "To summarize, the four most common data structures in agate ([`Column`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.Column), [`Row`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.Row), [`Table.columns`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.columns) and [`Table.rows`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.rows)) are all instances of [`MappedSequence`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.MappedSequence) and therefore all behave in a uniform way. This is also true of [`TableSet`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html), which will discuss later on." + "To summarize, the four most common data structures in agate ([`Column`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.Column), [`Row`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.Row), [`Table.columns`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.columns) and [`Table.rows`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.rows)) are all instances of [`MappedSequence`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.MappedSequence) and therefore all behave in a uniform way. This is also true of [`TableSet`](https://agate.readthedocs.io/en/latest/api/tableset.html), which will discuss later on." ] }, { @@ -464,11 +464,11 @@ "\n", "**Question:** How many exonerations involved a false confession?\n", "\n", - "Answering this question involves counting the number of ``True`` values in the ``false_confession`` column. When we created the table we specified that the data in this column contained [`Boolean`](http://agate.readthedocs.io/en/1.6.2/api/data_types.html#agate.Boolean) data. Because of this, agate has taken care of coercing the original text data from the CSV into Python's ``True`` and ``False`` values.\n", + "Answering this question involves counting the number of ``True`` values in the ``false_confession`` column. When we created the table we specified that the data in this column contained [`Boolean`](https://agate.readthedocs.io/en/latest/api/data_types.html#agate.Boolean) data. Because of this, agate has taken care of coercing the original text data from the CSV into Python's ``True`` and ``False`` values.\n", "\n", - "We'll answer the question using an instance of [`Count`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Count) which is a type of [`Aggregation`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Aggregation). Aggregations are used to perform \"column-wise\" calculations. That is, they derive a new single value from the contents of a column. The [`Count`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Count) aggregation can count either all values in a column, or how many times a particular value appears.\n", + "We'll answer the question using an instance of [`Count`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Count) which is a type of [`Aggregation`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Aggregation). Aggregations are used to perform \"column-wise\" calculations. That is, they derive a new single value from the contents of a column. The [`Count`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Count) aggregation can count either all values in a column, or how many times a particular value appears.\n", "\n", - "An Aggregation is applied to a table using [`Table.aggregate`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.aggregate).\n", + "An Aggregation is applied to a table using [`Table.aggregate`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.aggregate).\n", "\n", "It sounds complicated, but it's really simple. Putting it all together looks like this:" ] @@ -537,7 +537,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The answer to our question is \"26 years old\", however, as the warnings indicate, not every exonerated individual in the data has a value for the ``age`` column. The [`Median`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Median) statistical operation has no standard way of accounting for null values, so it leaves them out of the calculation.\n", + "The answer to our question is \"26 years old\", however, as the warnings indicate, not every exonerated individual in the data has a value for the ``age`` column. The [`Median`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Median) statistical operation has no standard way of accounting for null values, so it leaves them out of the calculation.\n", "\n", "**Question:** How many individuals do not have an age specified in the data?\n", "\n", @@ -574,7 +574,7 @@ "source": [ "Only nine rows in this dataset don't have age, so it's certainly still useful to compute a median. However, we might still want to filter those rows out so we could have a consistent sample for all of our calculations. In the next section you'll learn how to do just that.\n", "\n", - "Different [`aggregations`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html) can be applied depending on the type of data in each column. If none of the provided aggregations suit your needs you can use [`Summary`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Summary) to apply an arbitrary function to a column. If that still doesn't suit your needs you can always create your own aggregation from scratch by subclassing [`Aggregation`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Aggregation)." + "Different [`aggregations`](https://agate.readthedocs.io/en/latest/api/aggregations.html) can be applied depending on the type of data in each column. If none of the provided aggregations suit your needs you can use [`Summary`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Summary) to apply an arbitrary function to a column. If that still doesn't suit your needs you can always create your own aggregation from scratch by subclassing [`Aggregation`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Aggregation)." ] }, { @@ -584,9 +584,9 @@ "Selecting and filtering data\n", "============================\n", "\n", - "So what if those rows with no age were going to flummox our analysis? Agate's [`Table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table) class provides a full suite of SQL-like operations including [`Table.select`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.select) for grabbing specific columns, [`Table.where`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.where) for selecting particular rows and [`Table.group_by`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.group_by) for grouping rows by common values.\n", + "So what if those rows with no age were going to flummox our analysis? Agate's [`Table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table) class provides a full suite of SQL-like operations including [`Table.select`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.select) for grabbing specific columns, [`Table.where`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.where) for selecting particular rows and [`Table.group_by`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.group_by) for grouping rows by common values.\n", "\n", - "Let's use [`Table.where`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.where) to filter our exonerations table to only those individuals that have an age specified." + "Let's use [`Table.where`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.where) to filter our exonerations table to only those individuals that have an age specified." ] }, { @@ -604,9 +604,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You'll notice we provide a [`lambda`](https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions) function to the [`Table.where`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.where). This function is applied to each row and if it returns ``True``, then the row is included in the output table.\n", + "You'll notice we provide a [`lambda`](https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions) function to the [`Table.where`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.where). This function is applied to each row and if it returns ``True``, then the row is included in the output table.\n", "\n", - "A crucial thing to understand about these table methods is that they return **new tables**. In our example above ``exonerations`` was a [`Table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table) instance and we applied [`Table.where`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.where), so ``with_age`` is a new, different [`Table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table). The tables themselves can't be changed. You can create new tables with these methods, but you can't modify them in-place. (If this seems weird, just trust me. There are lots of good computer science-y reasons to do it this way.)\n", + "A crucial thing to understand about these table methods is that they return **new tables**. In our example above ``exonerations`` was a [`Table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table) instance and we applied [`Table.where`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.where), so ``with_age`` is a new, different [`Table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table). The tables themselves can't be changed. You can create new tables with these methods, but you can't modify them in-place. (If this seems weird, just trust me. There are lots of good computer science-y reasons to do it this way.)\n", "\n", "We can verify this did what we expected by counting the rows in the original table and rows in the new table:" ] @@ -671,13 +671,13 @@ "Computing new columns\n", "=====================\n", "\n", - "In addition to \"column-wise\" [`aggregations`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#module-agate.aggregations) there are also \"row-wise\" [`computations`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#module-agate.computations). Computations go through a [`Table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table) row-by-row and derive a new column using the existing data. To perform row computations in agate we use subclasses of [`Computation`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Computation).\n", + "In addition to \"column-wise\" [`aggregations`](https://agate.readthedocs.io/en/latest/api/aggregations.html#module-agate.aggregations) there are also \"row-wise\" [`computations`](https://agate.readthedocs.io/en/latest/api/computations.html#module-agate.computations). Computations go through a [`Table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table) row-by-row and derive a new column using the existing data. To perform row computations in agate we use subclasses of [`Computation`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Computation).\n", "\n", - "When one or more instances of [`Computation`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Computation) are applied with the [`Table.compute`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.compute) method, a new table is created with additional columns.\n", + "When one or more instances of [`Computation`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Computation) are applied with the [`Table.compute`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.compute) method, a new table is created with additional columns.\n", "\n", "**Question:** How long did individuals remain in prison before being exonerated?\n", "\n", - "To answer this question we will apply the [`Change`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Change) computation to the ``convicted`` and ``exonerated`` columns. Each of these columns contains the individual's age at the time of that event. All that [`Change`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Change) does is compute the difference between two numbers. (In this case each of these columns contain a [`Number`](http://agate.readthedocs.io/en/1.6.2/api/data_types.html#agate.Number), but this will also work with [`Date`](http://agate.readthedocs.io/en/1.6.2/api/data_types.html#agate.Date) or [`DateTime`](http://agate.readthedocs.io/en/1.6.2/api/data_types.html#agate.DateTime).)" + "To answer this question we will apply the [`Change`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Change) computation to the ``convicted`` and ``exonerated`` columns. Each of these columns contains the individual's age at the time of that event. All that [`Change`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Change) does is compute the difference between two numbers. (In this case each of these columns contain a [`Number`](https://agate.readthedocs.io/en/latest/api/data_types.html#agate.Number), but this will also work with [`Date`](https://agate.readthedocs.io/en/latest/api/data_types.html#agate.Date) or [`DateTime`](https://agate.readthedocs.io/en/latest/api/data_types.html#agate.DateTime).)" ] }, { @@ -712,7 +712,7 @@ "source": [ "The median number of years an exonerated individual spent in prison was 8 years.\n", "\n", - "Sometimes, the built-in computations, such as [`Change`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Change) won't suffice. I mentioned before that you could perform arbitrary column-wise aggregations using [`Summary`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Summary). You can do the same thing for row-wise computations using [`Formula`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Formula). This is somewhat analogous to Excel's cell formulas.\n", + "Sometimes, the built-in computations, such as [`Change`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Change) won't suffice. I mentioned before that you could perform arbitrary column-wise aggregations using [`Summary`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Summary). You can do the same thing for row-wise computations using [`Formula`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Formula). This is somewhat analogous to Excel's cell formulas.\n", "\n", "For example, this code will create a ``full_name`` column from the ``first_name`` and ``last_name`` columns in the data:" ] @@ -780,7 +780,7 @@ "source": [ "We add the ``replace`` argument to our ``compute`` method to replace the state column in place.\n", "\n", - "If [`Formula`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Formula) is not flexible enough (for instance, if you needed to compute a new value based on the distribution of data in a column) you can always implement your own subclass of [`Computation`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#agate.Computation). See the API documentation for [`computations`](http://agate.readthedocs.io/en/1.6.2/api/computations.html#module-agate.computations to see all of the supported ways to compute new data." + "If [`Formula`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Formula) is not flexible enough (for instance, if you needed to compute a new value based on the distribution of data in a column) you can always implement your own subclass of [`Computation`](https://agate.readthedocs.io/en/latest/api/computations.html#agate.Computation). See the API documentation for [`computations`](https://agate.readthedocs.io/en/latest/api/computations.html#module-agate.computations to see all of the supported ways to compute new data." ] }, { @@ -792,7 +792,7 @@ "\n", "**Question:** Who are the ten exonerated individuals who were youngest at the time they were arrested?\n", "\n", - "Remembering that methods of tables return tables, we will use [`Table.order_by`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.order_by) to sort our table:" + "Remembering that methods of tables return tables, we will use [`Table.order_by`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.order_by) to sort our table:" ] }, { @@ -810,7 +810,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can then use [`Table.limit`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.limit) get only the first ten rows of the data." + "We can then use [`Table.limit`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.limit) get only the first ten rows of the data." ] }, { @@ -828,7 +828,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now let's use [`Table.print_table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.print_table) to help us pretty the results in a way we can easily review:" + "Now let's use [`Table.print_table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.print_table) to help us pretty the results in a way we can easily review:" ] }, { @@ -867,9 +867,9 @@ "source": [ "If you find it impossible to believe that an eleven year-old was convicted of murder, I encourage you to read the Registry's [description of the case](http://www.law.umich.edu/special/exoneration/Pages/casedetail.aspx?caseid=3499>).\n", "\n", - "**Note:** In the previous example we could have omitted the [`Table.limit`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.limit) and passed a ``max_rows=10`` to [`Table.print_table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.print_table) instead. In this case they accomplish exactly the same goal.\n", + "**Note:** In the previous example we could have omitted the [`Table.limit`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.limit) and passed a ``max_rows=10`` to [`Table.print_table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.print_table) instead. In this case they accomplish exactly the same goal.\n", "\n", - "What if we were more curious about the *distribution* of ages, rather than the highest or lowest? agate includes the [`Table.pivot`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.pivot) and [`Table.bins`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.bins) methods for counting values individually or by ranges. Let's try binning the ages. Then, instead of using [`Table.print_table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.print_table), we'll use [`Table.print_bars`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.print_bars) to generate a simple, text bar chart." + "What if we were more curious about the *distribution* of ages, rather than the highest or lowest? agate includes the [`Table.pivot`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.pivot) and [`Table.bins`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.bins) methods for counting values individually or by ranges. Let's try binning the ages. Then, instead of using [`Table.print_table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.print_table), we'll use [`Table.print_bars`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.print_bars) to generate a simple, text bar chart." ] }, { @@ -925,7 +925,7 @@ "\n", "This question can't be answered by operating on a single column. What we need is the equivalent of SQL's ``GROUP BY``. agate supports a full set of SQL-like operations on tables. Unlike SQL, agate breaks grouping and aggregation into two discrete steps.\n", "\n", - "First, we use [`Table.group_by`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.group_by) to group the data by state." + "First, we use [`Table.group_by`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.group_by) to group the data by state." ] }, { @@ -943,9 +943,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This takes our original [`Table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table) and groups it into a [`TableSet`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet), which contains one table per state. As mentioned much earlier in this tutorial, TableSets are instances of [`MappedSequence`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.MappedSequence). That means they work very much like [`Column`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.Column) and [`Row`](http://agate.readthedocs.io/en/1.6.2/api/columns_and_rows.html#agate.Row).\n", + "This takes our original [`Table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table) and groups it into a [`TableSet`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet), which contains one table per state. As mentioned much earlier in this tutorial, TableSets are instances of [`MappedSequence`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.MappedSequence). That means they work very much like [`Column`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.Column) and [`Row`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.Row).\n", "\n", - "Now we need to aggregate the total for each state. This works in a very similar way to how it did when we were aggregating columns of a single table, except that we'll use the [`Count`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Count) aggregation to count the total number of rows in each group." + "Now we need to aggregate the total for each state. This works in a very similar way to how it did when we were aggregating columns of a single table, except that we'll use the [`Count`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Count) aggregation to count the total number of rows in each group." ] }, { @@ -984,11 +984,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You'll notice we pass a sequence of tuples to [`TableSet.aggregate`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet.aggregate). Each one includes two elements. The first is the new column name being created. The second is an instance of some [`Aggregation`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Aggregation). Unsurpringly, in this case the results appear to be roughly proportional to population.\n", + "You'll notice we pass a sequence of tuples to [`TableSet.aggregate`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet.aggregate). Each one includes two elements. The first is the new column name being created. The second is an instance of some [`Aggregation`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Aggregation). Unsurpringly, in this case the results appear to be roughly proportional to population.\n", "\n", "**Question:** What state has the longest median time in prison prior to exoneration?\n", "\n", - "This is a much more complicated question that's going to pull together a lot of the features we've been using. We'll repeat the computations we applied before, but this time we're going to roll those computations up in state-by-state groups and then take the [`Median`](http://agate.readthedocs.io/en/1.6.2/api/aggregations.html#agate.Median of each group. Then we'll sort the data and see where people have been stuck in prison the longest." + "This is a much more complicated question that's going to pull together a lot of the features we've been using. We'll repeat the computations we applied before, but this time we're going to roll those computations up in state-by-state groups and then take the [`Median`](https://agate.readthedocs.io/en/latest/api/aggregations.html#agate.Median of each group. Then we'll sort the data and see where people have been stuck in prison the longest." ] }, { @@ -1038,7 +1038,7 @@ "source": [ "DC? Nebraska? What accounts for these states having the longest times in prison before exoneration? I have no idea! Given that the group sizes are small, it would probably be wise to look for outliers.\n", "\n", - "As with [`Table.aggregate`]()http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.aggregate and [`Table.compute`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.compute), the [`TableSet.aggregate`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet.aggregate) method takes a list of aggregations to perform. You can aggregate as many columns as you like in a single step and they will all appear in the output table." + "As with [`Table.aggregate`]()https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.aggregate and [`Table.compute`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.compute), the [`TableSet.aggregate`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet.aggregate) method takes a list of aggregations to perform. You can aggregate as many columns as you like in a single step and they will all appear in the output table." ] }, { @@ -1048,9 +1048,9 @@ "Multi-dimensional aggregation\n", "=============================\n", "\n", - "I've already shown you that you can use [`TableSet`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet) to group instances of [`Table`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table). However, you can also use a [`TableSet`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet) to group *other TableSets*. To put that another way, instances of [`TableSet`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet) can be *nested*.\n", + "I've already shown you that you can use [`TableSet`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet) to group instances of [`Table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table). However, you can also use a [`TableSet`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet) to group *other TableSets*. To put that another way, instances of [`TableSet`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet) can be *nested*.\n", "\n", - "The key to nesting data in this way is to use [`TableSet.group_by`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet.group_by). This is one of many methods that can be called on a TableSet, which will then be applied to all the tables it contains. In the last section we used [`Table.group_by`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.group_by) to split data up into a group of tables. By calling [`TableSet.group_by`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet.group_by), which essentially called ``group_by`` on each table and collect the results. This can be pretty hard to wrap your head around, so let's look at a concrete example.\n", + "The key to nesting data in this way is to use [`TableSet.group_by`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet.group_by). This is one of many methods that can be called on a TableSet, which will then be applied to all the tables it contains. In the last section we used [`Table.group_by`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.group_by) to split data up into a group of tables. By calling [`TableSet.group_by`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet.group_by), which essentially called ``group_by`` on each table and collect the results. This can be pretty hard to wrap your head around, so let's look at a concrete example.\n", "\n", "**Question:** Is there a collective relationship between race, age and time spent in prison prior to exoneration?\n", "\n", @@ -1118,9 +1118,9 @@ "source": [ "## Exploratory charting\n", "\n", - "Beginning with version 1.5.0, agate includes the pure-Python SVG charting library [leather](http://leather.readthedocs.io/en/latest/). Leather allows you to generate \"good enough\" charts with as little as one line of code. It's especially useful if you're working in a Jupyter Notebook, as the results will render inline.\n", + "Beginning with version 1.5.0, agate includes the pure-Python SVG charting library [leather](https://leather.readthedocs.io/en/latest/). Leather allows you to generate \"good enough\" charts with as little as one line of code. It's especially useful if you're working in a Jupyter Notebook, as the results will render inline.\n", "\n", - "There are currently four chart types support: [`Table.bar_chart`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.bar_chart), [`Table.column_chart`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.column_chart), [`Table.line_chart`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.line_chart), and [`Table.scatterplot`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.scatterplot).\n", + "There are currently four chart types support: [`Table.bar_chart`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.bar_chart), [`Table.column_chart`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.column_chart), [`Table.line_chart`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.line_chart), and [`Table.scatterplot`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.scatterplot).\n", "\n", "Let's create charts from a few slices of data we've made in this tutorial." ] @@ -1170,7 +1170,7 @@ "source": [ "### Exonerations by age bracket\n", "\n", - "When creating a chart you may omit the column name arguments. If you do so the first and second columns in the table will be used. This is especially useful for charting the output of [`TableSet.aggregate`](http://agate.readthedocs.io/en/1.6.2/api/tableset.html#agate.TableSet.aggregate) or [`Table.bins`](http://agate.readthedocs.io/en/1.6.2/api/table.html#agate.Table.bins)." + "When creating a chart you may omit the column name arguments. If you do so the first and second columns in the table will be used. This is especially useful for charting the output of [`TableSet.aggregate`](https://agate.readthedocs.io/en/latest/api/tableset.html#agate.TableSet.aggregate) or [`Table.bins`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.bins)." ] }, { @@ -1293,7 +1293,7 @@ "source": [ "### Styling charts\n", "\n", - "As mentioned above, leather is designed for making \"good enough\" charts. You are never going to create a polished chart. However, sometimes you may want more control than agate offers through it's own methods. You can take more control over how your charts are presented by using [leather](http://leather.readthedocs.io/) directly." + "As mentioned above, leather is designed for making \"good enough\" charts. You are never going to create a polished chart. However, sometimes you may want more control than agate offers through it's own methods. You can take more control over how your charts are presented by using [leather](https://leather.readthedocs.io/) directly." ] }, { @@ -1336,9 +1336,9 @@ "Where to go next\n", "================\n", "\n", - "This tutorial only scratches the surface of agate's features. For many more ideas on how to apply agate, check out the [`Cookbook`](http://agate.readthedocs.io/en/1.6.2/cookbook.html), which includes dozens of examples of specific features of agate as well as recipes for substituting agate for Excel, SQL, R and more. Also check out the agate's [`Extensions`](http://agate.readthedocs.io/en/1.6.2/extensions.html) which add support for reading/writing SQL tables, performing statistical analysis and more.\n", + "This tutorial only scratches the surface of agate's features. For many more ideas on how to apply agate, check out the [`Cookbook`](https://agate.readthedocs.io/en/latest/cookbook.html), which includes dozens of examples of specific features of agate as well as recipes for substituting agate for Excel, SQL, R and more. Also check out the agate's [`Extensions`](https://agate.readthedocs.io/en/latest/extensions.html) which add support for reading/writing SQL tables, performing statistical analysis and more.\n", "\n", - "Also, if you're going to be doing data processing in Python you really ought to check out [`proof`](http://proof.readthedocs.org/en/1.6.2/), a library for building data processing pipelines that are repeatable and self-documenting. It will make your code cleaner and save you tons of time.\n", + "Also, if you're going to be doing data processing in Python you really ought to check out [`proof`](https://proof.readthedocs.io/en/0.3.0/), a library for building data processing pipelines that are repeatable and self-documenting. It will make your code cleaner and save you tons of time.\n", "\n", "Good luck in your reporting!" ] From 4f3597274ea30bb641f60989aa4ccce469b48ba6 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:50:26 -0400 Subject: [PATCH 039/163] docs: Add tags to recent changelog entries --- CHANGELOG.rst | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 95f58a14..4a8074cf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,21 +11,18 @@ 1.6.2 - March 10, 2021 ---------------------- -* :meth:`.Date.__init__` and :meth:`.DateTime.__init__` accepts a ``locale`` keyword argument (e.g. :code:`en_US`) for parsing formatted dates. (#730) -* :meth:`.utils.max_precision` ignores infinity when calculating precision. (#726) -* :meth:`.Date.cast` catches ``OverflowError`` when type testing. (#720) -* :meth:`.Number.cast` casts ``True`` to ``1`` and ``False`` to ``0``. (#733) +* feat: :meth:`.Date.__init__` and :meth:`.DateTime.__init__` accepts a ``locale`` keyword argument (e.g. :code:`en_US`) for parsing formatted dates. (#730) +* feat: :meth:`.Number.cast` casts ``True`` to ``1`` and ``False`` to ``0``. (#733) +* fix: :meth:`.utils.max_precision` ignores infinity when calculating precision. (#726) +* fix: :meth:`.Date.cast` catches ``OverflowError`` when type testing. (#720) * Included examples in Python package. (#716) 1.6.1 - March 11, 2018 ---------------------- -* :meth:`.Date.cast` and :meth:`.DateTime.cast` will no longer parse strings that contain dates as dates. (#705) -* Added Forest Gregg to Authors. -* :meth:`.Table.to_json` can now use Decimal as keys. (#696) -* Link to tutorial now uses version through sphinx to avoid bad links on future releases. (#682) -* lxml limited to >= 3.6 and < 4 for pypy compatibility. - +* feat: :meth:`.Table.to_json` can use Decimal as keys. (#696) +* fix: :meth:`.Date.cast` and :meth:`.DateTime.cast` no longer parse non-date strings that contain date sub-strings as dates. (#705) +* docs: Link to tutorial now uses version through Sphinx to avoid bad links on future releases. (#682) 1.6.0 - February 28, 2017 ------------------------- From 0667717b9849888cf172c3813eb5e524efbe698b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:51:25 -0400 Subject: [PATCH 040/163] docs: Simplify release process --- docs/release_process.rst | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/release_process.rst b/docs/release_process.rst index 0444b5e9..046b3157 100644 --- a/docs/release_process.rst +++ b/docs/release_process.rst @@ -2,17 +2,21 @@ Release process =============== -This is the release process for agate: +If substantial changes were made to the code: -#. Verify all tests pass on continuous integration. -3. Ensure any new modules have been added to setup.py's ``packages`` list. -#. Ensure any new public interfaces have been added to the documentation. -#. Ensure TableSet proxy methods have been added for new Table methods. -#. Run ``python charts.py`` to update images in the documentation. -#. Ensure ``CHANGELOG.rst`` is up to date. Add the release date and summary. -#. Create a release tag: ``git tag -a x.y.z -m "x.y.z release."`` -#. Push tags upstream: ``git push --follow-tags`` -#. Upload to `PyPI `_: ``python setup.py sdist bdist_wheel upload``. -#. Flag the release to build on `RTFD `_. -#. Rev to latest version: ``docs/conf.py``, ``setup.py``, ``CHANGELOG.rst`` need updates. -#. Commit revision: ``git commit -am "Update to version x.y.z for development."``. +#. Ensure any new modules have been added to setup.py's ``packages`` list +#. Ensure any new public interfaces have been added to the documentation +#. Ensure TableSet proxy methods have been added for new Table methods + +Then: + +#. All tests pass on continuous integration +#. The changelog is up-to-date and dated +#. The version number is correct in: + * setup.py + * docs/conf.py +#. Check for new authors: ``git log --invert-grep --author='James McKinney'`` +#. Run ``python charts.py`` to update images in the documentation +#. Tag the release: ``git tag -a x.y.z -m 'x.y.z release.'; git push --follow-tags`` +#. Upload to PyPI: ``rm -rf dist; python setup.py sdist bdist_wheel; twine upload dist/*`` +#. Build the documentation on ReadTheDocs manually From 53bccf7c0d96ced412dce258915f475c6e405efb Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:54:12 -0400 Subject: [PATCH 041/163] test: Relax performance for CI --- benchmarks/test_joins.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 58098e16..ee1056eb 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -16,7 +16,6 @@ import agate -@unittest.skipIf(platform.system() != "Linux") class TestTableJoin(unittest.TestCase): def test_join(self): left_rows = [(six.text_type(i), i) for i in range(100000)] @@ -38,4 +37,4 @@ def test(): min_time = min(results) - self.assertLess(min_time, 0) + self.assertLess(min_time, 2.5) # 2.5 for CI From 6397038e38430d453c7f160b757d89413824a616 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:55:32 -0400 Subject: [PATCH 042/163] chore: flake8 --- benchmarks/test_joins.py | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index ee1056eb..5133ff96 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import platform from random import shuffle from timeit import Timer From 277dec7335b5e8b56f11e0631380ec7a75675040 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 21:01:08 -0400 Subject: [PATCH 043/163] test: Set the locale, #712 --- tests/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..836f01ab 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +import locale + +# The test fixtures can break if the locale is non-US. +locale.setlocale(locale.LC_ALL, 'en_US') From 0cf9f8f2c4dc5a02230063f6f773a6241b3bf7c0 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 21:03:20 -0400 Subject: [PATCH 044/163] ci: Don't test example.py on Windows (UnicodeEncodeError) --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 973fa1ef..709009a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,5 +27,6 @@ jobs: - run: isort . --check-only - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate - - run: python example.py + - if: matrix.os != 'windows-latest' + run: python example.py - run: python charts.py From 5c37802ff2f3e90bfb85cf3850b2ae18e2234be0 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 21:07:08 -0400 Subject: [PATCH 045/163] ci: Exclude Windows on Python 3.6 and 3.7 --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 709009a0..fbd3a25e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,11 @@ jobs: matrix: os: [macos-latest, windows-latest, ubuntu-latest] python-version: [3.6, 3.7, 3.8, 3.9, pypy-3.6, pypy-3.7] + exclude: + - os: windows-latest + python-version: 3.6 + - os: windows-latest + python-version: 3.7 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 From 886db1a45d1b944c35d3d90640279e2dc300798a Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 22:39:41 -0400 Subject: [PATCH 046/163] ci: Generate locales on Linux --- .github/workflows/ci.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbd3a25e..43acec01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,10 @@ jobs: python-version: 3.6 - os: windows-latest python-version: 3.7 + - os: windows-latest + python-version: pypy-3.6 + - os: windows-latest + python-version: pypy-3.7 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -30,8 +34,15 @@ jobs: - run: check-manifest - run: flake8 . - run: isort . --check-only + - name: Set up locales + if: matrix.os == 'ubuntu-latest' + run: | + sudo locale-gen en_US.UTF-8 + sudo update-locale - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate - - if: matrix.os != 'windows-latest' - run: python example.py - - run: python charts.py + - name: Test example scripts + if: matrix.os != 'windows-latest' + run: | + python example.py + python charts.py From 6e6f906b47c1750f14e02b65cd825fd246a84c63 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 22:50:53 -0400 Subject: [PATCH 047/163] test: Set the locale to en_US.UTF-8 --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 836f01ab..ee8beb57 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ import locale # The test fixtures can break if the locale is non-US. -locale.setlocale(locale.LC_ALL, 'en_US') +locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') From f90f60ea451e73bdb2a124135b155d851a12f282 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 22:55:42 -0400 Subject: [PATCH 048/163] ci: Install libxml2-dev libxslt-dev on Ubuntu --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43acec01..db2a95ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,10 @@ jobs: - run: check-manifest - run: flake8 . - run: isort . --check-only - - name: Set up locales + - name: Install Ubuntu dependencies if: matrix.os == 'ubuntu-latest' run: | + sudo apt install libxml2-dev libxslt-dev sudo locale-gen en_US.UTF-8 sudo update-locale - run: pip install .[test] From 802b0a2931e249e6246d18d203fcf81bae730909 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:14:49 -0400 Subject: [PATCH 049/163] ci: Generate all locales used in tests --- .github/workflows/ci.yml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db2a95ba..783e3e7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ jobs: os: [macos-latest, windows-latest, ubuntu-latest] python-version: [3.6, 3.7, 3.8, 3.9, pypy-3.6, pypy-3.7] exclude: + # UnicodeDecodeError on test_to_csv - os: windows-latest python-version: 3.6 - os: windows-latest @@ -17,6 +18,14 @@ jobs: - os: windows-latest python-version: pypy-3.7 steps: + - if: matrix.os == 'ubuntu-latest' + name: Install Ubuntu dependencies + run: | + sudo apt install libxml2-dev libxslt-dev + sudo locale-gen en_US.UTF-8 + sudo locale-gen de_DE + sudo locale-gen ko_KR + sudo update-locale - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: @@ -34,16 +43,10 @@ jobs: - run: check-manifest - run: flake8 . - run: isort . --check-only - - name: Install Ubuntu dependencies - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt install libxml2-dev libxslt-dev - sudo locale-gen en_US.UTF-8 - sudo update-locale - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate - - name: Test example scripts - if: matrix.os != 'windows-latest' - run: | - python example.py - python charts.py + # UnicodeDecodeError on print_bars + - if: matrix.os != 'windows-latest' + name: Test example scripts + run: python example.py + - run: python charts.py From 0349efe6e80d2e1ad9bb538d90d52b901a1a422f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:21:49 -0400 Subject: [PATCH 050/163] build: Remove unittest2 --- .github/workflows/ci.yml | 7 ++++--- agate/testcase.py | 5 +---- benchmarks/test_joins.py | 5 +---- setup.py | 1 - tests/__init__.py | 2 +- tests/test_agate.py | 5 +---- tests/test_aggregations.py | 5 +---- tests/test_columns.py | 5 +---- tests/test_computations.py | 5 +---- tests/test_data_types.py | 5 +---- tests/test_fixed.py | 5 +---- tests/test_mapped_sequence.py | 5 +---- tests/test_py2.py | 5 +---- tests/test_py3.py | 5 +---- tests/test_type_tester.py | 5 +---- tests/test_utils.py | 5 +---- 16 files changed, 18 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 783e3e7b..a9547117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,10 @@ jobs: # UnicodeDecodeError on test_to_csv - os: windows-latest python-version: 3.6 - - os: windows-latest - python-version: 3.7 - os: windows-latest python-version: pypy-3.6 + - os: windows-latest + python-version: 3.7 - os: windows-latest python-version: pypy-3.7 steps: @@ -22,8 +22,9 @@ jobs: name: Install Ubuntu dependencies run: | sudo apt install libxml2-dev libxslt-dev - sudo locale-gen en_US.UTF-8 sudo locale-gen de_DE + sudo locale-gen en_US + sudo locale-gen fr_FR sudo locale-gen ko_KR sudo update-locale - uses: actions/checkout@v2 diff --git a/agate/testcase.py b/agate/testcase.py index 89438200..71db94ee 100644 --- a/agate/testcase.py +++ b/agate/testcase.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import agate diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 5133ff96..da0ca60b 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -4,10 +4,7 @@ from random import shuffle from timeit import Timer -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import six from six.moves import range diff --git a/setup.py b/setup.py index 2c3ed024..a8695829 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,6 @@ 'PyICU>=2.4.2;sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', - 'unittest2>=1.1.0;python_version<"3"', ], 'docs': [ 'Sphinx>=1.2.2', diff --git a/tests/__init__.py b/tests/__init__.py index ee8beb57..836f01ab 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ import locale # The test fixtures can break if the locale is non-US. -locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') +locale.setlocale(locale.LC_ALL, 'en_US') diff --git a/tests/test_agate.py b/tests/test_agate.py index cd6e4fcc..b0c84d42 100644 --- a/tests/test_agate.py +++ b/tests/test_agate.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import six diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 09279afe..93760050 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -6,10 +6,7 @@ import warnings from decimal import Decimal -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate import Table from agate.aggregations import (IQR, MAD, All, Any, Count, Deciles, First, HasNulls, Max, MaxLength, MaxPrecision, diff --git a/tests/test_columns.py b/tests/test_columns.py index 107219e7..a0cfb233 100644 --- a/tests/test_columns.py +++ b/tests/test_columns.py @@ -8,10 +8,7 @@ except ImportError: # pragma: no cover from decimal import Decimal -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_computations.py b/tests/test_computations.py index c94482a1..84922fcc 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -4,10 +4,7 @@ import warnings from decimal import Decimal -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate import Table from agate.computations import Change, Formula, Percent, PercentChange, PercentileRank, Rank, Slug diff --git a/tests/test_data_types.py b/tests/test_data_types.py index e0d7f377..ad579e84 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -8,10 +8,7 @@ import parsedatetime import six -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import pytz diff --git a/tests/test_fixed.py b/tests/test_fixed.py index 5ce42b56..18973d6e 100644 --- a/tests/test_fixed.py +++ b/tests/test_fixed.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate import csv, fixed diff --git a/tests/test_mapped_sequence.py b/tests/test_mapped_sequence.py index b113ecc6..5ff271c2 100644 --- a/tests/test_mapped_sequence.py +++ b/tests/test_mapped_sequence.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import six diff --git a/tests/test_py2.py b/tests/test_py2.py index d1e384e8..d066aaa0 100644 --- a/tests/test_py2.py +++ b/tests/test_py2.py @@ -4,10 +4,7 @@ import csv import os -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import six diff --git a/tests/test_py3.py b/tests/test_py3.py index 29c68a96..e9f3da60 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -6,10 +6,7 @@ import six -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate import csv_py3 from agate.exceptions import FieldSizeLimitError diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index e8d801ff..0837f174 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -1,10 +1,7 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.type_tester import TypeTester diff --git a/tests/test_utils.py b/tests/test_utils.py index 9df54fcb..40e84627 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,10 +5,7 @@ except ImportError: # pragma: no cover from decimal import Decimal -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate.utils import Quantiles, letter_name, round_limits From 4fb91e6c5e58c643d4608674aa11f3178fdb9d37 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:23:45 -0400 Subject: [PATCH 051/163] chore: isort --- benchmarks/test_joins.py | 3 +-- tests/test_aggregations.py | 3 +-- tests/test_computations.py | 3 +-- tests/test_data_types.py | 6 ++---- tests/test_py2.py | 1 - tests/test_py3.py | 3 +-- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index da0ca60b..96b9a952 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -1,11 +1,10 @@ #!/usr/bin/env python # -*- coding: utf8 -*- +import unittest from random import shuffle from timeit import Timer -import unittest - import six from six.moves import range diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 93760050..2e4ada49 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -3,11 +3,10 @@ import datetime import sys +import unittest import warnings from decimal import Decimal -import unittest - from agate import Table from agate.aggregations import (IQR, MAD, All, Any, Count, Deciles, First, HasNulls, Max, MaxLength, MaxPrecision, Mean, Median, Min, Mode, Percentiles, PopulationStDev, PopulationVariance, Quartiles, diff --git a/tests/test_computations.py b/tests/test_computations.py index 84922fcc..e0aa3b75 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -1,11 +1,10 @@ #!/usr/bin/env Python import datetime +import unittest import warnings from decimal import Decimal -import unittest - from agate import Table from agate.computations import Change, Formula, Percent, PercentChange, PercentileRank, Rank, Slug from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta diff --git a/tests/test_data_types.py b/tests/test_data_types.py index ad579e84..b41e76e7 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -3,14 +3,12 @@ import datetime import pickle +import unittest from decimal import Decimal import parsedatetime -import six - -import unittest - import pytz +import six from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.exceptions import CastError diff --git a/tests/test_py2.py b/tests/test_py2.py index d066aaa0..cb296146 100644 --- a/tests/test_py2.py +++ b/tests/test_py2.py @@ -3,7 +3,6 @@ import csv import os - import unittest import six diff --git a/tests/test_py3.py b/tests/test_py3.py index e9f3da60..572365aa 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -3,11 +3,10 @@ import csv import os +import unittest import six -import unittest - from agate import csv_py3 from agate.exceptions import FieldSizeLimitError From 98fe1767e61dc62bf4f330b54a27510baa2b907f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:28:43 -0400 Subject: [PATCH 052/163] test: Add .UTF-8 to locales --- .github/workflows/ci.yml | 8 ++++---- tests/test_data_types.py | 10 +++++----- tests/test_table/test_print_bars.py | 2 +- tests/test_table/test_print_structure.py | 2 +- tests/test_table/test_print_table.py | 4 ++-- tests/test_type_tester.py | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9547117..65b23e1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,10 +22,10 @@ jobs: name: Install Ubuntu dependencies run: | sudo apt install libxml2-dev libxslt-dev - sudo locale-gen de_DE - sudo locale-gen en_US - sudo locale-gen fr_FR - sudo locale-gen ko_KR + sudo locale-gen de_DE.UTF-8 + sudo locale-gen en_US.UTF-8 + sudo locale-gen fr_FR.UTF-8 + sudo locale-gen ko_KR.UTF-8 sudo update-locale - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/tests/test_data_types.py b/tests/test_data_types.py index b41e76e7..6c930bfa 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -164,7 +164,7 @@ def test_currency_cast(self): def test_cast_locale(self): values = (2, 1, None, Decimal('2.7'), 'n/a', '2,7', '200.000.000') - casted = tuple(Number(locale='de_DE').cast(v) for v in values) + casted = tuple(Number(locale='de_DE.UTF-8').cast(v) for v in values) self.assertSequenceEqual(casted, (Decimal('2'), Decimal('1'), None, Decimal('2.7'), None, Decimal('2.7'), Decimal('200000000'))) def test_cast_text(self): @@ -251,7 +251,7 @@ def test_cast_format(self): )) def test_cast_format_locale(self): - date_type = Date(date_format='%d-%b-%Y', locale='de_DE') + date_type = Date(date_format='%d-%b-%Y', locale='de_DE.UTF-8') # March can be abbreviated to Mrz or Mär depending on the locale version, # so we use December in the first value to ensure the test passes everywhere @@ -266,7 +266,7 @@ def test_cast_format_locale(self): )) def test_cast_locale(self): - date_type = Date(locale='fr_FR') + date_type = Date(locale='fr_FR.UTF-8') values = ('01 mars 1994', u'jeudi 17 février 2011', None, '5 janvier 1984', 'n/a') casted = tuple(date_type.cast(v) for v in values) @@ -375,7 +375,7 @@ def test_cast_format(self): )) def test_cast_format_locale(self): - date_type = DateTime(datetime_format='%Y-%m-%d %I:%M %p', locale='ko_KR') + date_type = DateTime(datetime_format='%Y-%m-%d %I:%M %p', locale='ko_KR.UTF-8') # Date formats depend on the platform's strftime/strptime implementation; # some platforms like macOS always return AM/PM for day periods (%p), @@ -404,7 +404,7 @@ def test_cast_format_locale(self): raise AssertionError('\n\n'.join(exceptions)) def test_cast_locale(self): - date_type = DateTime(locale='fr_FR') + date_type = DateTime(locale='fr_FR.UTF-8') values = ('01/03/1994 12:30', '17/2/11 6:30', None, '5/01/84 18:30', 'n/a') casted = tuple(date_type.cast(v) for v in values) diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index 0c31d503..83b6557d 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -19,7 +19,7 @@ def setUp(self): ) self.number_type = Number() - self.international_number_type = Number(locale='de_DE') + self.international_number_type = Number(locale='de_DE.UTF-8') self.text_type = Text() self.column_names = ['one', 'two', 'three'] diff --git a/tests/test_table/test_print_structure.py b/tests/test_table/test_print_structure.py index 18b47adf..e60a84fe 100644 --- a/tests/test_table/test_print_structure.py +++ b/tests/test_table/test_print_structure.py @@ -17,7 +17,7 @@ def setUp(self): ) self.number_type = Number() - self.international_number_type = Number(locale='de_DE') + self.international_number_type = Number(locale='de_DE.UTF-8') self.text_type = Text() self.column_names = ['one', 'two', 'three'] diff --git a/tests/test_table/test_print_table.py b/tests/test_table/test_print_table.py index 1d4aef76..aebb0242 100644 --- a/tests/test_table/test_print_table.py +++ b/tests/test_table/test_print_table.py @@ -19,7 +19,7 @@ def setUp(self): self.number_type = Number() self.american_number_type = Number(locale='en_US') - self.german_number_type = Number(locale='de_DE') + self.german_number_type = Number(locale='de_DE.UTF-8') self.text_type = Text() self.column_names = ['one', 'two', 'three', 'four'] @@ -130,6 +130,6 @@ def test_print_table_locale_german(self): table = Table(self.rows, self.column_names, self.column_types) output = six.StringIO() - table.print_table(max_columns=2, output=output, locale='de_DE') + table.print_table(max_columns=2, output=output, locale='de_DE.UTF-8') # If it's working, the english '2,000' should appear as '2.000' self.assertTrue("2.000" in output.getvalue()) diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 0837f174..2f932204 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -199,8 +199,8 @@ def test_types_number_locale(self): ('',) ] - tester = TypeTester(types=[Number(locale='de_DE'), Text()]) + tester = TypeTester(types=[Number(locale='de_DE.UTF-8'), Text()]) inferred = tester.run(rows, ['one']) self.assertIsInstance(inferred[0], Number) - self.assertEqual(str(inferred[0].locale), 'de_DE') + self.assertEqual(str(inferred[0].locale), 'de_DE.UTF-8') From a37b6a8419b0aaf8d2eff06891a9fe5014a5ec4d Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:33:25 -0400 Subject: [PATCH 053/163] test: Attempt to get tests to pass in CI --- .github/workflows/ci.yml | 3 +-- tests/test_data_types.py | 4 ++-- tests/test_type_tester.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65b23e1e..b8ce851a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,12 +19,11 @@ jobs: python-version: pypy-3.7 steps: - if: matrix.os == 'ubuntu-latest' - name: Install Ubuntu dependencies + name: Install UTF-8 locales and lxml requirements run: | sudo apt install libxml2-dev libxslt-dev sudo locale-gen de_DE.UTF-8 sudo locale-gen en_US.UTF-8 - sudo locale-gen fr_FR.UTF-8 sudo locale-gen ko_KR.UTF-8 sudo update-locale - uses: actions/checkout@v2 diff --git a/tests/test_data_types.py b/tests/test_data_types.py index 6c930bfa..eaf60806 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -266,7 +266,7 @@ def test_cast_format_locale(self): )) def test_cast_locale(self): - date_type = Date(locale='fr_FR.UTF-8') + date_type = Date(locale='fr_FR') values = ('01 mars 1994', u'jeudi 17 février 2011', None, '5 janvier 1984', 'n/a') casted = tuple(date_type.cast(v) for v in values) @@ -404,7 +404,7 @@ def test_cast_format_locale(self): raise AssertionError('\n\n'.join(exceptions)) def test_cast_locale(self): - date_type = DateTime(locale='fr_FR.UTF-8') + date_type = DateTime(locale='fr_FR') values = ('01/03/1994 12:30', '17/2/11 6:30', None, '5/01/84 18:30', 'n/a') casted = tuple(date_type.cast(v) for v in values) diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 2f932204..231947ca 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -203,4 +203,4 @@ def test_types_number_locale(self): inferred = tester.run(rows, ['one']) self.assertIsInstance(inferred[0], Number) - self.assertEqual(str(inferred[0].locale), 'de_DE.UTF-8') + self.assertEqual(str(inferred[0].locale), 'de_DE') From fb7dc6126eb0df706b7d60c63c4ee095edf7c92a Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:38:02 -0400 Subject: [PATCH 054/163] test: Fix default locale string --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 836f01ab..ee8beb57 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ import locale # The test fixtures can break if the locale is non-US. -locale.setlocale(locale.LC_ALL, 'en_US') +locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') From 777a06d522d517f0d9c2cd84f3658b645ca18fc1 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:43:26 -0400 Subject: [PATCH 055/163] test: Relax performance for CI --- benchmarks/test_joins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 96b9a952..8027bb81 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -32,4 +32,4 @@ def test(): min_time = min(results) - self.assertLess(min_time, 2.5) # 2.5 for CI + self.assertLess(min_time, 3) # 3 for CI From b52ff97b5cfed5b6520608d0d6355edad587ed4b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:47:34 -0400 Subject: [PATCH 056/163] test: Relax performance for CI (again) --- benchmarks/test_joins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 8027bb81..25db9277 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -32,4 +32,4 @@ def test(): min_time = min(results) - self.assertLess(min_time, 3) # 3 for CI + self.assertLess(min_time, 4) # 4 for CI From 9e728d70d5b734b6f0d4e51bb05eee868242263d Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jul 2021 23:54:42 -0400 Subject: [PATCH 057/163] test: Relax performance for CI (again) --- benchmarks/test_joins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 25db9277..c2ffdf68 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -32,4 +32,4 @@ def test(): min_time = min(results) - self.assertLess(min_time, 4) # 4 for CI + self.assertLess(min_time, 10) # CI unreliable, 5 witnessed From abf824e153e330afec9ece4635d8767f4ef8fa9f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 00:18:03 -0400 Subject: [PATCH 058/163] ci: Attempt Python 2.7 tests --- .github/workflows/ci.yml | 6 +++++- benchmarks/test_joins.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8ce851a..0ff0e2af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,12 @@ jobs: strategy: matrix: os: [macos-latest, windows-latest, ubuntu-latest] - python-version: [3.6, 3.7, 3.8, 3.9, pypy-3.6, pypy-3.7] + python-version: [2.7, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7] exclude: + - os: windows-latest + python-version: 2.7 + - os: windows-latest + python-version: pypy-2.7 # UnicodeDecodeError on test_to_csv - os: windows-latest python-version: 3.6 diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index c2ffdf68..924dc1d9 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -32,4 +32,4 @@ def test(): min_time = min(results) - self.assertLess(min_time, 10) # CI unreliable, 5 witnessed + self.assertLess(min_time, 10) # CI unreliable, 5s witnessed From 548a49295071f6de481c32cd85bb9fc1009f1c5b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 00:23:21 -0400 Subject: [PATCH 059/163] build: Make PyICU an optional dependency on Python 3+ --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a8695829..ff6e6446 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0', 'nose>=1.1.2', - 'PyICU>=2.4.2;sys_platform=="linux"', + 'PyICU>=2.4.2;python_version>"2";sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', ], From 2b677b647d4544cb37d534af126454281dd6aa56 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 00:33:44 -0400 Subject: [PATCH 060/163] build: Fix version specifier --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff6e6446..90ea0851 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0', 'nose>=1.1.2', - 'PyICU>=2.4.2;python_version>"2";sys_platform=="linux"', + 'PyICU>=2.4.2;python_version>"2",sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', ], From cee2378331e0212beed6b00465536b0b4f528de9 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 00:38:50 -0400 Subject: [PATCH 061/163] build: Fix version specifier --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90ea0851..dd53ed57 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0', 'nose>=1.1.2', - 'PyICU>=2.4.2;python_version>"2",sys_platform=="linux"', + 'PyICU>=2.4.2;python_version>"2" and sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', ], From 748867b3f8c7b662de45e97c9b501cca0e49c9a4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:02:06 -0400 Subject: [PATCH 062/163] build: parsedatetime==2.6 has a bug on Python 2.7 --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index dd53ed57..8ffc2af7 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,9 @@ 'Babel>=2.0', 'isodate>=0.5.4', 'leather>=0.3.2', - 'parsedatetime>=2.1,!=2.5', + # KeyError: 's' https://github.com/bear/parsedatetime/pull/233 https://github.com/wireservice/agate/issues/743 + # AttributeError: 'module' object has no attribute 'Locale' https://github.com/bear/parsedatetime/pull/247 + 'parsedatetime>=2.1,!=2.5,!=2.6', 'python-slugify>=1.2.1', 'pytimeparse>=1.1.5', 'six>=1.9.0', @@ -47,7 +49,7 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0', 'nose>=1.1.2', - 'PyICU>=2.4.2;python_version>"2" and sys_platform=="linux"', + 'PyICU>=2.4.2;sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', ], From ba83a194f292338992e62322ff98178913dbf4fa Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:19:39 -0400 Subject: [PATCH 063/163] Revert "build: Remove unittest2" as assertWarns was added in 3.2 This reverts commit 0349efe6e80d2e1ad9bb538d90d52b901a1a422f. --- .github/workflows/ci.yml | 4 ++-- agate/testcase.py | 5 ++++- benchmarks/test_joins.py | 6 +++++- setup.py | 1 + tests/test_agate.py | 5 ++++- tests/test_aggregations.py | 6 +++++- tests/test_columns.py | 5 ++++- tests/test_computations.py | 6 +++++- tests/test_data_types.py | 6 +++++- tests/test_fixed.py | 5 ++++- tests/test_mapped_sequence.py | 5 ++++- tests/test_py2.py | 6 +++++- tests/test_py3.py | 6 +++++- tests/test_type_tester.py | 5 ++++- tests/test_utils.py | 5 ++++- 15 files changed, 61 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ff0e2af..592d8f11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,10 @@ jobs: # UnicodeDecodeError on test_to_csv - os: windows-latest python-version: 3.6 - - os: windows-latest - python-version: pypy-3.6 - os: windows-latest python-version: 3.7 + - os: windows-latest + python-version: pypy-3.6 - os: windows-latest python-version: pypy-3.7 steps: diff --git a/agate/testcase.py b/agate/testcase.py index 71db94ee..89438200 100644 --- a/agate/testcase.py +++ b/agate/testcase.py @@ -1,6 +1,9 @@ #!/usr/bin/env python -import unittest +try: + import unittest2 as unittest +except ImportError: + import unittest import agate diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 924dc1d9..b26d0da7 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -1,10 +1,14 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import unittest from random import shuffle from timeit import Timer +try: + import unittest2 as unittest +except ImportError: + import unittest + import six from six.moves import range diff --git a/setup.py b/setup.py index 8ffc2af7..3f447755 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ 'PyICU>=2.4.2;sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', + 'unittest2>=1.1.0;python_version<"3"', ], 'docs': [ 'Sphinx>=1.2.2', diff --git a/tests/test_agate.py b/tests/test_agate.py index b0c84d42..cd6e4fcc 100644 --- a/tests/test_agate.py +++ b/tests/test_agate.py @@ -1,6 +1,9 @@ #!/usr/bin/env python -import unittest +try: + import unittest2 as unittest +except ImportError: + import unittest import six diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 2e4ada49..09279afe 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -3,10 +3,14 @@ import datetime import sys -import unittest import warnings from decimal import Decimal +try: + import unittest2 as unittest +except ImportError: + import unittest + from agate import Table from agate.aggregations import (IQR, MAD, All, Any, Count, Deciles, First, HasNulls, Max, MaxLength, MaxPrecision, Mean, Median, Min, Mode, Percentiles, PopulationStDev, PopulationVariance, Quartiles, diff --git a/tests/test_columns.py b/tests/test_columns.py index a0cfb233..107219e7 100644 --- a/tests/test_columns.py +++ b/tests/test_columns.py @@ -8,7 +8,10 @@ except ImportError: # pragma: no cover from decimal import Decimal -import unittest +try: + import unittest2 as unittest +except ImportError: + import unittest from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_computations.py b/tests/test_computations.py index e0aa3b75..c94482a1 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -1,10 +1,14 @@ #!/usr/bin/env Python import datetime -import unittest import warnings from decimal import Decimal +try: + import unittest2 as unittest +except ImportError: + import unittest + from agate import Table from agate.computations import Change, Formula, Percent, PercentChange, PercentileRank, Rank, Slug from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta diff --git a/tests/test_data_types.py b/tests/test_data_types.py index eaf60806..cd574b23 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -3,9 +3,13 @@ import datetime import pickle -import unittest from decimal import Decimal +try: + import unittest2 as unittest +except ImportError: + import unittest + import parsedatetime import pytz import six diff --git a/tests/test_fixed.py b/tests/test_fixed.py index 18973d6e..5ce42b56 100644 --- a/tests/test_fixed.py +++ b/tests/test_fixed.py @@ -1,6 +1,9 @@ #!/usr/bin/env python -import unittest +try: + import unittest2 as unittest +except ImportError: + import unittest from agate import csv, fixed diff --git a/tests/test_mapped_sequence.py b/tests/test_mapped_sequence.py index 5ff271c2..b113ecc6 100644 --- a/tests/test_mapped_sequence.py +++ b/tests/test_mapped_sequence.py @@ -1,6 +1,9 @@ #!/usr/bin/env python -import unittest +try: + import unittest2 as unittest +except ImportError: + import unittest import six diff --git a/tests/test_py2.py b/tests/test_py2.py index cb296146..d1e384e8 100644 --- a/tests/test_py2.py +++ b/tests/test_py2.py @@ -3,7 +3,11 @@ import csv import os -import unittest + +try: + import unittest2 as unittest +except ImportError: + import unittest import six diff --git a/tests/test_py3.py b/tests/test_py3.py index 572365aa..8d5ab750 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -3,7 +3,11 @@ import csv import os -import unittest + +try: + import unittest2 as unittest +except ImportError: + import unittest import six diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 231947ca..4c23481a 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -1,7 +1,10 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import unittest +try: + import unittest2 as unittest +except ImportError: + import unittest from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.type_tester import TypeTester diff --git a/tests/test_utils.py b/tests/test_utils.py index 40e84627..9df54fcb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,10 @@ except ImportError: # pragma: no cover from decimal import Decimal -import unittest +try: + import unittest2 as unittest +except ImportError: + import unittest from agate.utils import Quantiles, letter_name, round_limits From 89455200daab6936371e10248951a99b98aac40f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:25:58 -0400 Subject: [PATCH 064/163] ci: Skip testing example on Python 2.7 --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 592d8f11..2d2c3fe8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,6 @@ jobs: - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate # UnicodeDecodeError on print_bars - - if: matrix.os != 'windows-latest' - name: Test example scripts + - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' run: python example.py - run: python charts.py From db938a922bce4a994745f0802d38e23953acd9bb Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:30:17 -0400 Subject: [PATCH 065/163] ci: Skip testing example on PyPy 2.7. Attempt testing on Python 2.7 on all platforms. --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d2c3fe8..8ce98f6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,10 +8,6 @@ jobs: os: [macos-latest, windows-latest, ubuntu-latest] python-version: [2.7, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7] exclude: - - os: windows-latest - python-version: 2.7 - - os: windows-latest - python-version: pypy-2.7 # UnicodeDecodeError on test_to_csv - os: windows-latest python-version: 3.6 @@ -50,6 +46,6 @@ jobs: - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate # UnicodeDecodeError on print_bars - - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' + - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' run: python example.py - run: python charts.py From dfaab767fe645258ef61a8b7722d2f40d287fdac Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:34:45 -0400 Subject: [PATCH 066/163] ci: Exclude Windows on Python 2.7 --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ce98f6d..55e4361e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,11 +10,15 @@ jobs: exclude: # UnicodeDecodeError on test_to_csv - os: windows-latest - python-version: 3.6 + python-version: 2.7 - os: windows-latest - python-version: 3.7 + python-version: pypy-2.7 + - os: windows-latest + python-version: 3.6 - os: windows-latest python-version: pypy-3.6 + - os: windows-latest + python-version: 3.7 - os: windows-latest python-version: pypy-3.7 steps: From aff92a8428d8f51803caf2832567939415a9b063 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:59:39 -0400 Subject: [PATCH 067/163] ci: Attempt testing PyICU on macOS and Windows --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f447755..f29ea573 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0', 'nose>=1.1.2', - 'PyICU>=2.4.2;sys_platform=="linux"', + 'PyICU>=2.4.2', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', 'unittest2>=1.1.0;python_version<"3"', From 35780db5ba49f3be9325bbd0ec054ea309a52934 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 12:06:17 -0400 Subject: [PATCH 068/163] ci: Exclude PyICU on macOS and Windows --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f29ea573..d992880a 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,8 @@ 'cssselect>=0.9.1', 'lxml>=3.6.0', 'nose>=1.1.2', - 'PyICU>=2.4.2', + # CI is not configured to install PyICU on macOS and Windows. + 'PyICU>=2.4.2;sys_platform=="linux"', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', 'unittest2>=1.1.0;python_version<"3"', From c4edc3107ab578c3d7fd70bf8492b3e36f31f7c9 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 13:08:53 -0400 Subject: [PATCH 069/163] Remove fragment sentence, closes #667 --- AUTHORS.rst | 3 ++- CHANGELOG.rst | 1 + agate/table/from_csv.py | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 0f79cdbe..7f9b8275 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -43,4 +43,5 @@ agate is made by a community. The following individuals have contributed code, d * `Loïc Corbasson `_ * `Danny Sepler `_ * `brian-from-quantrocket `_ - +* `mathdesc `_ +* `Tim Gates `_ diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a8074cf..83943adf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,7 @@ ------------------ * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) +* feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) * feat: :class:`.Max` works with :class:`.TimeDelta`. (#735) * fix: :class:`.Mean` returns ``None`` if there are no values to average. (#706) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index d95cead0..5a88d2b4 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -26,8 +26,7 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk :param row_names: See :meth:`.Table.__init__`. :param skip_lines: - The number of lines to skip from the top of the file. Note that skip - lines will not work with + The number of lines to skip from the top of the file. :param header: If :code:`True`, the first row of the CSV is assumed to contain column names. If :code:`header` and :code:`column_names` are both specified From f235b6ad8dfdaa80ca518e406e8f33c6c8f137e2 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 13:19:09 -0400 Subject: [PATCH 070/163] feat: Add line number to FieldSizeLimitError message, closes #681 --- CHANGELOG.rst | 1 + agate/csv_py2.py | 2 +- agate/csv_py3.py | 2 +- agate/exceptions.py | 5 +++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 83943adf..d0089636 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,7 @@ * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) * feat: :class:`.Max` works with :class:`.TimeDelta`. (#735) +* feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) * fix: :class:`.Mean` returns ``None`` if there are no values to average. (#706) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) * fix: Ensure files are closed when errors occur. (#734) diff --git a/agate/csv_py2.py b/agate/csv_py2.py index 32a113b5..546aa5dd 100644 --- a/agate/csv_py2.py +++ b/agate/csv_py2.py @@ -54,7 +54,7 @@ def next(self): except csv.Error as e: # Terrible way to test for this exception, but there is no subclass if 'field larger than field limit' in str(e): - raise FieldSizeLimitError(csv.field_size_limit()) + raise FieldSizeLimitError(csv.field_size_limit(), csv.line_num) else: raise e diff --git a/agate/csv_py3.py b/agate/csv_py3.py index 918af60a..e7183182 100644 --- a/agate/csv_py3.py +++ b/agate/csv_py3.py @@ -35,7 +35,7 @@ def __next__(self): except csv.Error as e: # Terrible way to test for this exception, but there is no subclass if 'field larger than field limit' in str(e): - raise FieldSizeLimitError(csv.field_size_limit()) + raise FieldSizeLimitError(csv.field_size_limit(), self.line_num) else: raise e diff --git a/agate/exceptions.py b/agate/exceptions.py index 7fbfcc29..0673c7c4 100644 --- a/agate/exceptions.py +++ b/agate/exceptions.py @@ -34,7 +34,8 @@ class FieldSizeLimitError(Exception): # pragma: no cover This length may be the default or one set by the user. """ - def __init__(self, limit): + def __init__(self, limit, line_number): super(FieldSizeLimitError, self).__init__( - 'CSV contains fields longer than maximum length of %i characters. Try raising the maximum with the field_size_limit parameter, or try setting quoting=csv.QUOTE_NONE.' % limit + 'CSV contains a field longer than the maximum length of %i characters on line %i. Try raising the maximum ' + 'with the field_size_limit parameter, or try setting quoting=csv.QUOTE_NONE.' % (limit, line_number) ) From a22c98ec494d172c2d906daf7ca0dd57cbb1d121 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 13:33:18 -0400 Subject: [PATCH 071/163] test: Add test case related to #371 --- tests/test_type_tester.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 4c23481a..56d8f96e 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -14,6 +14,18 @@ class TestTypeTester(unittest.TestCase): def setUp(self): self.tester = TypeTester() + def test_empty(self): + rows = [ + (None,), + (None,), + (None,), + ] + + inferred = self.tester.run(rows, ['one']) + + # This behavior is not necessarily desirable. See https://github.com/wireservice/agate/issues/371 + self.assertIsInstance(inferred[0], Boolean) + def test_text_type(self): rows = [ ('a',), From e6563c111ee82856cee03531f3e2ee936c48c64f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 13:38:14 -0400 Subject: [PATCH 072/163] feat: Fix typo in f235b6ad8dfdaa80ca518e406e8f33c6c8f137e2 --- agate/csv_py2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agate/csv_py2.py b/agate/csv_py2.py index 546aa5dd..e84c627f 100644 --- a/agate/csv_py2.py +++ b/agate/csv_py2.py @@ -54,7 +54,7 @@ def next(self): except csv.Error as e: # Terrible way to test for this exception, but there is no subclass if 'field larger than field limit' in str(e): - raise FieldSizeLimitError(csv.field_size_limit(), csv.line_num) + raise FieldSizeLimitError(csv.field_size_limit(), self.line_num) else: raise e From 8fa5e6efaac60e7c0a2bdab61e2300e1ac906769 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 14:20:02 -0400 Subject: [PATCH 073/163] test: Avoid noqa by not assigning variable in assertRaises block --- tests/test_computations.py | 10 +++++----- tests/test_from_json.py | 2 +- tests/test_table/__init__.py | 22 +++++++++++----------- tests/test_table/test_join.py | 4 ++-- tests/test_table/test_merge.py | 2 +- tests/test_table/test_pivot.py | 2 +- tests/test_tableset/__init__.py | 6 +++--- tests/test_tableset/test_merge.py | 4 ++-- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/test_computations.py b/tests/test_computations.py index c94482a1..df1857e0 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -54,7 +54,7 @@ def test_formula(self): def test_formula_invalid(self): with self.assertRaises(CastError): - new_table = self.table.compute([ # noqa + self.table.compute([ ('test', Formula(self.number_type, lambda r: r['one'])) ]) @@ -179,11 +179,11 @@ def to_one_place(d): self.assertEqual(to_one_place(new_table.columns['test'][3]), Decimal('60.0')) with self.assertRaises(DataTypeError): - new_table = self.table.compute([ + self.table.compute([ ('test', Percent('two', 0)) ]) with self.assertRaises(DataTypeError): - new_table = self.table.compute([ + self.table.compute([ ('test', Percent('two', -1)) ]) with self.assertRaises(DataTypeError): @@ -250,12 +250,12 @@ def to_one_place(d): def test_percent_change_invalid_columns(self): with self.assertRaises(DataTypeError): - new_table = self.table.compute([ + self.table.compute([ ('test', PercentChange('one', 'three')) ]) with self.assertRaises(DataTypeError): - new_table = self.table.compute([ # noqa + self.table.compute([ ('test', PercentChange('three', 'one')) ]) diff --git a/tests/test_from_json.py b/tests/test_from_json.py index 5dfd4fb5..471dccc8 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -93,4 +93,4 @@ def test_from_json_no_type_tester(self): def test_from_json_error_newline_key(self): with self.assertRaises(ValueError): - table = Table.from_json('examples/test.json', newline=True, key='test') # noqa + Table.from_json('examples/test.json', newline=True, key='test') diff --git a/tests/test_table/__init__.py b/tests/test_table/__init__.py index d60a0d27..bd86a450 100644 --- a/tests/test_table/__init__.py +++ b/tests/test_table/__init__.py @@ -46,7 +46,7 @@ def test_create_table(self): def test_create_filename(self): with self.assertRaises(ValueError): - table = Table('foo.csv') # noqa + Table('foo.csv') def test_create_empty_table(self): table = Table([]) @@ -118,7 +118,7 @@ def test_create_table_cast_error(self): column_types = [self.number_type, self.number_type, self.number_type] with self.assertRaises(CastError) as e: - table = Table(self.rows, self.column_names, column_types) # noqa + Table(self.rows, self.column_names, column_types) self.assertIn('Error at row 0 column three.', str(e.exception)) @@ -126,31 +126,31 @@ def test_create_table_null_column_names(self): column_names = ['one', None, 'three'] with self.assertWarns(RuntimeWarning): - table2 = Table(self.rows, column_names, self.column_types) # noqa + Table(self.rows, column_names, self.column_types) warnings.simplefilter('ignore') try: - table3 = Table(self.rows, column_names, self.column_types) + table = Table(self.rows, column_names, self.column_types) finally: warnings.resetwarnings() - self.assertColumnNames(table3, ['one', 'b', 'three']) + self.assertColumnNames(table, ['one', 'b', 'three']) def test_create_table_empty_column_names(self): column_names = ['one', '', 'three'] with self.assertWarns(RuntimeWarning): - table2 = Table(self.rows, column_names, self.column_types) # noqa + Table(self.rows, column_names, self.column_types) warnings.simplefilter('ignore') try: - table3 = Table(self.rows, column_names, self.column_types) + table = Table(self.rows, column_names, self.column_types) finally: warnings.resetwarnings() - self.assertColumnNames(table3, ['one', 'b', 'three']) + self.assertColumnNames(table, ['one', 'b', 'three']) def test_create_table_non_datatype_columns(self): column_types = [self.number_type, self.number_type, 'foo'] @@ -253,7 +253,7 @@ def test_row_too_long(self): ) with self.assertRaises(ValueError): - table = Table(rows, self.column_names, self.column_types) # noqa + Table(rows, self.column_names, self.column_types) def test_row_names(self): table = Table(self.rows, self.column_names, self.column_types, row_names='three') @@ -274,7 +274,7 @@ def test_row_names_non_string(self): def test_row_names_int(self): with self.assertRaises(ValueError): - table = Table(self.rows, self.column_names, self.column_types, row_names=['a', 'b', 3]) # noqa + Table(self.rows, self.column_names, self.column_types, row_names=['a', 'b', 3]) def test_row_names_func(self): table = Table(self.rows, self.column_names, self.column_types, row_names=lambda r: (r['one'], r['three'])) @@ -291,7 +291,7 @@ def test_row_names_func(self): def test_row_names_invalid(self): with self.assertRaises(ValueError): - table = Table( # noqa + Table( self.rows, self.column_names, self.column_types, diff --git a/tests/test_table/test_join.py b/tests/test_table/test_join.py index f4991c65..f7655c61 100644 --- a/tests/test_table/test_join.py +++ b/tests/test_table/test_join.py @@ -213,10 +213,10 @@ def test_join_with_row_names(self): def test_join_require_match(self): with self.assertRaises(ValueError): - new_table = self.left.join(self.right, 'one', 'five', require_match=True) # noqa + self.left.join(self.right, 'one', 'five', require_match=True) with self.assertRaises(ValueError): - new_table = self.left.join(self.right, 'one', 'five', require_match=True) # noqa + self.left.join(self.right, 'one', 'five', require_match=True) new_table = self.left.join(self.right, 'one', 'four', require_match=True) # noqa diff --git a/tests/test_table/test_merge.py b/tests/test_table/test_merge.py index 626b9a18..f2f84812 100644 --- a/tests/test_table/test_merge.py +++ b/tests/test_table/test_merge.py @@ -76,7 +76,7 @@ def test_merge_different_types(self): table_b = Table(self.rows, self.column_names, column_types) with self.assertRaises(DataTypeError): - table_c = Table.merge([table_a, table_b]) # noqa + Table.merge([table_a, table_b]) def test_merge_with_row_names(self): table_a = Table(self.rows, self.column_names, self.column_types, row_names='three') diff --git a/tests/test_table/test_pivot.py b/tests/test_table/test_pivot.py index 7ac90526..72ef4fcc 100644 --- a/tests/test_table/test_pivot.py +++ b/tests/test_table/test_pivot.py @@ -83,7 +83,7 @@ def test_pivot_by_lambda_group_name_sequence_invalid(self): table = Table(self.rows, self.column_names, self.column_types) with self.assertRaises(ValueError): - pivot_table = table.pivot(['race', 'gender'], key_name='foo') # noqa + table.pivot(['race', 'gender'], key_name='foo') def test_pivot_no_key(self): table = Table(self.rows, self.column_names, self.column_types) diff --git a/tests/test_tableset/__init__.py b/tests/test_tableset/__init__.py index b4b1616c..a1a8c54c 100644 --- a/tests/test_tableset/__init__.py +++ b/tests/test_tableset/__init__.py @@ -61,7 +61,7 @@ def test_create_tableset_mismatched_column_names(self): ]) with self.assertRaises(ValueError): - tableset = TableSet(tables.values(), tables.keys()) # noqa + TableSet(tables.values(), tables.keys()) def test_create_tableset_mismatched_column_types(self): tables = OrderedDict([ @@ -71,7 +71,7 @@ def test_create_tableset_mismatched_column_types(self): ]) with self.assertRaises(ValueError): - tableset = TableSet(tables.values(), tables.keys()) # noqa + TableSet(tables.values(), tables.keys()) def test_iter(self): tableset = TableSet(self.tables.values(), self.tables.keys()) @@ -161,7 +161,7 @@ def test_from_json_file(self): def test_from_json_false_path(self): with self.assertRaises(IOError): - tableset1 = TableSet.from_json('notapath') # noqa + TableSet.from_json('notapath') def test_to_json(self): tableset = TableSet(self.tables.values(), self.tables.keys()) diff --git a/tests/test_tableset/test_merge.py b/tests/test_tableset/test_merge.py index 6c3077db..bc1ee533 100644 --- a/tests/test_tableset/test_merge.py +++ b/tests/test_tableset/test_merge.py @@ -75,10 +75,10 @@ def test_merge_groups_invalid_length(self): tableset = TableSet(self.tables.values(), self.tables.keys()) with self.assertRaises(ValueError): - table = tableset.merge(groups=['red', 'blue'], group_name='color_code') # noqa + tableset.merge(groups=['red', 'blue'], group_name='color_code') def test_merge_groups_invalid_type(self): tableset = TableSet(self.tables.values(), self.tables.keys()) with self.assertRaises(ValueError): - table = tableset.merge(groups='invalid', group_name='color_code') # noqa + tableset.merge(groups='invalid', group_name='color_code') From 938b5f40d67b9008bf505c93a7a52e7280772a08 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 14:20:55 -0400 Subject: [PATCH 074/163] chore: Remove more noqa comments --- agate/aggregations/__init__.py | 46 +++++++++++++++++----------------- agate/computations/__init__.py | 16 ++++++------ agate/data_types/__init__.py | 16 ++++++------ setup.cfg | 3 +++ 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/agate/aggregations/__init__.py b/agate/aggregations/__init__.py index f12b3195..01d9a9cc 100644 --- a/agate/aggregations/__init__.py +++ b/agate/aggregations/__init__.py @@ -15,26 +15,26 @@ with a column for each aggregation and a row for each table in the set. """ -from agate.aggregations.all import All # noqa -from agate.aggregations.any import Any # noqa -from agate.aggregations.base import Aggregation # noqa -from agate.aggregations.count import Count # noqa -from agate.aggregations.deciles import Deciles # noqa -from agate.aggregations.first import First # noqa -from agate.aggregations.has_nulls import HasNulls # noqa -from agate.aggregations.iqr import IQR # noqa -from agate.aggregations.mad import MAD # noqa -from agate.aggregations.max import Max # noqa -from agate.aggregations.max_length import MaxLength # noqa -from agate.aggregations.max_precision import MaxPrecision # noqa -from agate.aggregations.mean import Mean # noqa -from agate.aggregations.median import Median # noqa -from agate.aggregations.min import Min # noqa -from agate.aggregations.mode import Mode # noqa -from agate.aggregations.percentiles import Percentiles # noqa -from agate.aggregations.quartiles import Quartiles # noqa -from agate.aggregations.quintiles import Quintiles # noqa -from agate.aggregations.stdev import PopulationStDev, StDev # noqa -from agate.aggregations.sum import Sum # noqa -from agate.aggregations.summary import Summary # noqa -from agate.aggregations.variance import PopulationVariance, Variance # noqa +from agate.aggregations.all import All +from agate.aggregations.any import Any +from agate.aggregations.base import Aggregation +from agate.aggregations.count import Count +from agate.aggregations.deciles import Deciles +from agate.aggregations.first import First +from agate.aggregations.has_nulls import HasNulls +from agate.aggregations.iqr import IQR +from agate.aggregations.mad import MAD +from agate.aggregations.max import Max +from agate.aggregations.max_length import MaxLength +from agate.aggregations.max_precision import MaxPrecision +from agate.aggregations.mean import Mean +from agate.aggregations.median import Median +from agate.aggregations.min import Min +from agate.aggregations.mode import Mode +from agate.aggregations.percentiles import Percentiles +from agate.aggregations.quartiles import Quartiles +from agate.aggregations.quintiles import Quintiles +from agate.aggregations.stdev import PopulationStDev, StDev +from agate.aggregations.sum import Sum +from agate.aggregations.summary import Summary +from agate.aggregations.variance import PopulationVariance, Variance diff --git a/agate/computations/__init__.py b/agate/computations/__init__.py index 666f620d..3639215e 100644 --- a/agate/computations/__init__.py +++ b/agate/computations/__init__.py @@ -13,11 +13,11 @@ class by inheriting from :class:`Computation`. """ -from agate.computations.base import Computation # noqa -from agate.computations.change import Change # noqa -from agate.computations.formula import Formula # noqa -from agate.computations.percent import Percent # noqa -from agate.computations.percent_change import PercentChange # noqa -from agate.computations.percentile_rank import PercentileRank # noqa -from agate.computations.rank import Rank # noqa -from agate.computations.slug import Slug # noqa +from agate.computations.base import Computation +from agate.computations.change import Change +from agate.computations.formula import Formula +from agate.computations.percent import Percent +from agate.computations.percent_change import PercentChange +from agate.computations.percentile_rank import PercentileRank +from agate.computations.rank import Rank +from agate.computations.slug import Slug diff --git a/agate/data_types/__init__.py b/agate/data_types/__init__.py index 73e61d5a..6b91e278 100644 --- a/agate/data_types/__init__.py +++ b/agate/data_types/__init__.py @@ -9,11 +9,11 @@ control how types are guessed. """ -from agate.data_types.base import DEFAULT_NULL_VALUES, DataType # noqa -from agate.data_types.boolean import DEFAULT_FALSE_VALUES, DEFAULT_TRUE_VALUES, Boolean # noqa -from agate.data_types.date import Date # noqa -from agate.data_types.date_time import DateTime # noqa -from agate.data_types.number import Number # noqa -from agate.data_types.text import Text # noqa -from agate.data_types.time_delta import TimeDelta # noqa -from agate.exceptions import CastError # noqa +from agate.data_types.base import DEFAULT_NULL_VALUES, DataType +from agate.data_types.boolean import DEFAULT_FALSE_VALUES, DEFAULT_TRUE_VALUES, Boolean +from agate.data_types.date import Date +from agate.data_types.date_time import DateTime +from agate.data_types.number import Number +from agate.data_types.text import Text +from agate.data_types.time_delta import TimeDelta +from agate.exceptions import CastError diff --git a/setup.cfg b/setup.cfg index 8b71880f..378d4a7d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,9 @@ extend-ignore = E501 per-file-ignores = # imported but unused, unable to detect undefined names agate/__init__.py: F401,F403 + agate/aggregations/__init__.py: F401 + agate/computations/__init__.py: F401 + agate/data_types/__init__.py: F401 # module level import not at top of file agate/tableset/__init__.py: E402 agate/table/__init__.py: E402 From 8a3b0a588f62e05b7638647aa1362471bcb756db Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 14:34:04 -0400 Subject: [PATCH 075/163] fix: Aggregations return None if all values are None, closes #714 --- CHANGELOG.rst | 2 +- agate/aggregations/iqr.py | 3 +- agate/aggregations/mad.py | 6 +- agate/aggregations/max.py | 4 +- agate/aggregations/mean.py | 11 +-- agate/aggregations/min.py | 4 +- agate/aggregations/mode.py | 9 +- agate/aggregations/percentiles.py | 3 + agate/aggregations/stdev.py | 8 +- agate/aggregations/variance.py | 12 +-- agate/utils.py | 3 + tests/test_aggregations.py | 144 ++++++++++++++++++++---------- 12 files changed, 137 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d0089636..f3e8e9b2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) * feat: :class:`.Max` works with :class:`.TimeDelta`. (#735) * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) -* fix: :class:`.Mean` returns ``None`` if there are no values to average. (#706) +* fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) * fix: Ensure files are closed when errors occur. (#734) * build: Make PyICU an optional dependency. diff --git a/agate/aggregations/iqr.py b/agate/aggregations/iqr.py index 81d7c73a..cc902f50 100644 --- a/agate/aggregations/iqr.py +++ b/agate/aggregations/iqr.py @@ -36,4 +36,5 @@ def validate(self, table): def run(self, table): percentiles = self._percentiles.run(table) - return percentiles[75] - percentiles[25] + if percentiles[75] is not None and percentiles[25] is not None: + return percentiles[75] - percentiles[25] diff --git a/agate/aggregations/mad.py b/agate/aggregations/mad.py index 122db1b7..8fe9e2ce 100644 --- a/agate/aggregations/mad.py +++ b/agate/aggregations/mad.py @@ -39,6 +39,6 @@ def run(self, table): column = table.columns[self._column_name] data = column.values_without_nulls_sorted() - m = self._median.run(table) - - return median(tuple(abs(n - m) for n in data)) + if data: + m = self._median.run(table) + return median(tuple(abs(n - m) for n in data)) diff --git a/agate/aggregations/max.py b/agate/aggregations/max.py index 11b7cec5..fe83d7b2 100644 --- a/agate/aggregations/max.py +++ b/agate/aggregations/max.py @@ -33,4 +33,6 @@ def validate(self, table): def run(self, table): column = table.columns[self._column_name] - return max(column.values_without_nulls()) + data = column.values_without_nulls() + if data: + return max(data) diff --git a/agate/aggregations/mean.py b/agate/aggregations/mean.py index d2de20c9..689b674f 100644 --- a/agate/aggregations/mean.py +++ b/agate/aggregations/mean.py @@ -35,10 +35,7 @@ def validate(self, table): def run(self, table): column = table.columns[self._column_name] - num_of_values = len(column.values_without_nulls()) - # If there are no non-null columns then return null. - if num_of_values == 0: - return None - - sum_total = self._sum.run(table) - return sum_total / num_of_values + data = column.values_without_nulls() + if data: + sum_total = self._sum.run(table) + return sum_total / len(data) diff --git a/agate/aggregations/min.py b/agate/aggregations/min.py index 974dfbda..f5ab042e 100644 --- a/agate/aggregations/min.py +++ b/agate/aggregations/min.py @@ -33,4 +33,6 @@ def validate(self, table): def run(self, table): column = table.columns[self._column_name] - return min(column.values_without_nulls()) + data = column.values_without_nulls() + if data: + return min(data) diff --git a/agate/aggregations/mode.py b/agate/aggregations/mode.py index d9aa1ac3..0deb9295 100644 --- a/agate/aggregations/mode.py +++ b/agate/aggregations/mode.py @@ -37,9 +37,10 @@ def run(self, table): column = table.columns[self._column_name] data = column.values_without_nulls() - state = defaultdict(int) + if data: + state = defaultdict(int) - for n in data: - state[n] += 1 + for n in data: + state[n] += 1 - return max(state.keys(), key=lambda x: state[x]) + return max(state.keys(), key=lambda x: state[x]) diff --git a/agate/aggregations/percentiles.py b/agate/aggregations/percentiles.py index 56b6ec19..1a7c27c4 100644 --- a/agate/aggregations/percentiles.py +++ b/agate/aggregations/percentiles.py @@ -51,6 +51,9 @@ def run(self, table): data = column.values_without_nulls_sorted() + if not data: + return Quantiles([None for percentile in range(101)]) + # Zeroth percentile is first datum quantiles = [data[0]] diff --git a/agate/aggregations/stdev.py b/agate/aggregations/stdev.py index 21f03ab5..1b27c421 100644 --- a/agate/aggregations/stdev.py +++ b/agate/aggregations/stdev.py @@ -36,7 +36,9 @@ def validate(self, table): warn_null_calculation(self, column) def run(self, table): - return self._variance.run(table).sqrt() + variance = self._variance.run(table) + if variance is not None: + return variance.sqrt() class PopulationStDev(StDev): @@ -67,4 +69,6 @@ def validate(self, table): warn_null_calculation(self, column) def run(self, table): - return self._population_variance.run(table).sqrt() + variance = self._population_variance.run(table) + if variance is not None: + return variance.sqrt() diff --git a/agate/aggregations/variance.py b/agate/aggregations/variance.py index 0dbc4d08..0cdb7a93 100644 --- a/agate/aggregations/variance.py +++ b/agate/aggregations/variance.py @@ -39,9 +39,9 @@ def run(self, table): column = table.columns[self._column_name] data = column.values_without_nulls() - mean = self._mean.run(table) - - return sum((n - mean) ** 2 for n in data) / (len(data) - 1) + if data: + mean = self._mean.run(table) + return sum((n - mean) ** 2 for n in data) / (len(data) - 1) class PopulationVariance(Variance): @@ -75,6 +75,6 @@ def run(self, table): column = table.columns[self._column_name] data = column.values_without_nulls() - mean = self._mean.run(table) - - return sum((n - mean) ** 2 for n in data) / len(data) + if data: + mean = self._mean.run(table) + return sum((n - mean) ** 2 for n in data) / len(data) diff --git a/agate/utils.py b/agate/utils.py index 1ce98175..3272ab73 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -86,6 +86,9 @@ def __len__(self): def __repr__(self): return repr(self._quantiles) + def __eq__(self, other): + return self._quantiles == other._quantiles + def locate(self, value): """ Identify which quantile a given value is part of. diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 09279afe..8f053986 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -17,6 +17,7 @@ Quintiles, StDev, Sum, Summary, Variance) from agate.data_types import Boolean, DateTime, Number, Text, TimeDelta from agate.exceptions import DataTypeError +from agate.utils import Quantiles from agate.warns import NullCalculationWarning @@ -180,43 +181,45 @@ def test_all(self): class TestDateTimeAggregation(unittest.TestCase): - def test_min(self): - rows = [ + def setUp(self): + self.rows = [ [datetime.datetime(1994, 3, 3, 6, 31)], [datetime.datetime(1994, 3, 3, 6, 30, 30)], [datetime.datetime(1994, 3, 3, 6, 30)], ] - table = Table(rows, ['test'], [DateTime()]) - - self.assertIsInstance(Min('test').get_aggregate_data_type(table), DateTime) - Min('test').validate(table) - self.assertEqual(Min('test').run(table), datetime.datetime(1994, 3, 3, 6, 30)) + self.table = Table(self.rows, ['test', 'null'], [DateTime(), DateTime()]) - def test_max(self): - rows = [ - [datetime.datetime(1994, 3, 3, 6, 31)], - [datetime.datetime(1994, 3, 3, 6, 30, 30)], - [datetime.datetime(1994, 3, 3, 6, 30)], + self.time_delta_rows = [ + [datetime.timedelta(seconds=10), None], + [datetime.timedelta(seconds=20), None], ] - table = Table(rows, ['test'], [DateTime()]) + self.time_delta_table = Table(self.time_delta_rows, ['test', 'null'], [TimeDelta(), TimeDelta()]) - self.assertIsInstance(Max('test').get_aggregate_data_type(table), DateTime) - Max('test').validate(table) - self.assertEqual(Max('test').run(table), datetime.datetime(1994, 3, 3, 6, 31)) + def test_min(self): + self.assertIsInstance(Min('test').get_aggregate_data_type(self.table), DateTime) + Min('test').validate(self.table) + self.assertEqual(Min('test').run(self.table), datetime.datetime(1994, 3, 3, 6, 30)) - def test_sum(self): - rows = [ - [datetime.timedelta(seconds=10)], - [datetime.timedelta(seconds=20)], - ] + def test_min_all_nulls(self): + self.assertIsNone(Min('null').run(self.table)) - table = Table(rows, ['test'], [TimeDelta()]) + def test_max(self): + self.assertIsInstance(Max('test').get_aggregate_data_type(self.table), DateTime) + Max('test').validate(self.table) + self.assertEqual(Max('test').run(self.table), datetime.datetime(1994, 3, 3, 6, 31)) + + def test_max_all_nulls(self): + self.assertIsNone(Max('null').run(self.table)) + + def test_sum(self): + self.assertIsInstance(Sum('test').get_aggregate_data_type(self.time_delta_table), TimeDelta) + Sum('test').validate(self.time_delta_table) + self.assertEqual(Sum('test').run(self.time_delta_table), datetime.timedelta(seconds=30)) - self.assertIsInstance(Sum('test').get_aggregate_data_type(table), TimeDelta) - Sum('test').validate(table) - self.assertEqual(Sum('test').run(table), datetime.timedelta(seconds=30)) + def test_sum_all_nulls(self): + self.assertEqual(Sum('null').run(self.time_delta_table), datetime.timedelta(0)) class TestNumberAggregation(unittest.TestCase): @@ -245,6 +248,9 @@ def test_max_precision(self): self.assertEqual(MaxPrecision('one').run(self.table), 1) self.assertEqual(MaxPrecision('two').run(self.table), 2) + def test_max_precision_all_nulls(self): + self.assertEqual(MaxPrecision('four').run(self.table), 0) + def test_sum(self): with self.assertRaises(DataTypeError): Sum('three').validate(self.table) @@ -254,6 +260,9 @@ def test_sum(self): self.assertEqual(Sum('one').run(self.table), Decimal('6.5')) self.assertEqual(Sum('two').run(self.table), Decimal('13.13')) + def test_sum_all_nulls(self): + self.assertEqual(Sum('four').run(self.table), Decimal('0')) + def test_min(self): with self.assertRaises(DataTypeError): Min('three').validate(self.table) @@ -263,6 +272,9 @@ def test_min(self): self.assertEqual(Min('one').run(self.table), Decimal('1.1')) self.assertEqual(Min('two').run(self.table), Decimal('2.19')) + def test_min_all_nulls(self): + self.assertIsNone(Min('four').run(self.table)) + def test_max(self): with self.assertRaises(DataTypeError): Max('three').validate(self.table) @@ -272,6 +284,9 @@ def test_max(self): self.assertEqual(Max('one').run(self.table), Decimal('2.7')) self.assertEqual(Max('two').run(self.table), Decimal('4.1')) + def test_max_all_nulls(self): + self.assertIsNone(Max('four').run(self.table)) + def test_mean(self): with self.assertWarns(NullCalculationWarning): Mean('one').validate(self.table) @@ -284,14 +299,6 @@ def test_mean(self): self.assertEqual(Mean('two').run(self.table), Decimal('3.2825')) def test_mean_all_nulls(self): - """ - Test to confirm mean of only nulls doesn't cause a critical error. - - The assumption here is that if you attempt to perform a mean - calculation, on a column which contains only null values, then a null - value should be returned to the caller. - :return: - """ self.assertIsNone(Mean('four').run(self.table)) def test_mean_with_nulls(self): @@ -321,6 +328,9 @@ def test_median(self): self.assertIsInstance(Median('two').get_aggregate_data_type(self.table), Number) self.assertEqual(Median('two').run(self.table), Decimal('3.42')) + def test_median_all_nulls(self): + self.assertIsNone(Median('four').run(self.table)) + def test_mode(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -341,6 +351,9 @@ def test_mode(self): self.assertIsInstance(Mode('two').get_aggregate_data_type(self.table), Number) self.assertEqual(Mode('two').run(self.table), Decimal('3.42')) + def test_mode_all_nulls(self): + self.assertIsNone(Mode('four').run(self.table)) + def test_iqr(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -361,6 +374,9 @@ def test_iqr(self): self.assertIsInstance(IQR('two').get_aggregate_data_type(self.table), Number) self.assertEqual(IQR('two').run(self.table), Decimal('0.955')) + def test_irq_all_nulls(self): + self.assertIsNone(IQR('four').run(self.table)) + def test_variance(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -384,6 +400,9 @@ def test_variance(self): Decimal('0.6332') ) + def test_variance_all_nulls(self): + self.assertIsNone(Variance('four').run(self.table)) + def test_population_variance(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -407,6 +426,9 @@ def test_population_variance(self): Decimal('0.4749') ) + def test_population_variance_all_nulls(self): + self.assertIsNone(PopulationVariance('four').run(self.table)) + def test_stdev(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -430,6 +452,9 @@ def test_stdev(self): Decimal('0.7958') ) + def test_stdev_all_nulls(self): + self.assertIsNone(StDev('four').run(self.table)) + def test_population_stdev(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -453,6 +478,9 @@ def test_population_stdev(self): Decimal('0.6891') ) + def test_population_stdev_all_nulls(self): + self.assertIsNone(PopulationStDev('four').run(self.table)) + def test_mad(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -473,6 +501,9 @@ def test_mad(self): self.assertIsInstance(MAD('two').get_aggregate_data_type(self.table), Number) self.assertAlmostEqual(MAD('two').run(self.table), Decimal('0')) + def test_mad_all_nulls(self): + self.assertIsNone(MAD('four').run(self.table)) + def test_percentiles(self): with warnings.catch_warnings(): warnings.simplefilter('error') @@ -503,6 +534,9 @@ def test_percentiles(self): self.assertEqual(percentiles[99], Decimal('990.5')) self.assertEqual(percentiles[100], Decimal('1000')) + def test_percentiles_all_nulls(self): + self.assertEqual(Percentiles('four').run(self.table), Quantiles([None] * 101)) + def test_percentiles_locate(self): rows = [(n,) for n in range(1, 1001)] @@ -621,6 +655,9 @@ def test_quartiles(self): for i, v in enumerate(['1', '2', '4', '6', '7']): self.assertEqual(quartiles[i], Decimal(v)) + def test_quartiles_all_nulls(self): + self.assertEqual(Quartiles('four').run(self.table), Quantiles([None] * 5)) + def test_quartiles_locate(self): """ CDF quartile tests from: @@ -661,11 +698,16 @@ def test_quintiles(self): finally: warnings.resetwarnings() - rows = [(n,) for n in range(1, 1001)] + rows = [(n,) for n in range(1, 1000)] table = Table(rows, ['ints'], [self.number_type]) - quintiles = Quintiles('ints').run(table) # noqa + quintiles = Quintiles('ints').run(table) + for i, v in enumerate(['1', '200', '400', '600', '800', '999']): + self.assertEqual(quintiles[i], Decimal(v)) + + def test_quintiles_all_nulls(self): + self.assertEqual(Quintiles('four').run(self.table), Quantiles([None] * 6)) def test_deciles(self): with warnings.catch_warnings(): @@ -684,25 +726,35 @@ def test_deciles(self): finally: warnings.resetwarnings() - rows = [(n,) for n in range(1, 1001)] + rows = [(n,) for n in range(1, 1000)] table = Table(rows, ['ints'], [self.number_type]) - deciles = Deciles('ints').run(table) # noqa + deciles = Deciles('ints').run(table) + for i, v in enumerate(['1', '100', '200', '300', '400', '500', '600', '700', '800', '900', '999']): + self.assertEqual(deciles[i], Decimal(v)) + + def test_deciles_all_nulls(self): + self.assertEqual(Deciles('four').run(self.table), Quantiles([None] * 11)) class TestTextAggregation(unittest.TestCase): - def test_max_length(self): - rows = [ - ['a'], - ['gobble'], - ['w'] + def setUp(self): + self.rows = [ + ['a', None], + ['gobble', None], + ['w', None] ] - table = Table(rows, ['test'], [Text()]) - MaxLength('test').validate(table) - self.assertEqual(MaxLength('test').run(table), 6) - self.assertIsInstance(MaxLength('test').run(table), Decimal) + self.table = Table(self.rows, ['test', 'null'], [Text(), Text()]) + + def test_max_length(self): + MaxLength('test').validate(self.table) + self.assertEqual(MaxLength('test').run(self.table), 6) + self.assertIsInstance(MaxLength('test').run(self.table), Decimal) + + def test_max_length_all_nulls(self): + self.assertEqual(MaxLength('null').run(self.table), 0) def test_max_length_unicode(self): """ From 591654d8f247ee8252f046d11b0cec8fc9a579fe Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 14:57:55 -0400 Subject: [PATCH 076/163] fix: TableSet.group_by accepts input with no rows, closes #703 --- CHANGELOG.rst | 1 + agate/table/group_by.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3e8e9b2..9070d4aa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) * feat: :class:`.Max` works with :class:`.TimeDelta`. (#735) * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) +* feat: :meth:`.TableSet.group_by` accepts input with no rows. (#703) * fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) * fix: Ensure files are closed when errors occur. (#734) diff --git a/agate/table/group_by.py b/agate/table/group_by.py index 0a7d8e23..b5c6a91c 100644 --- a/agate/table/group_by.py +++ b/agate/table/group_by.py @@ -55,6 +55,9 @@ def group_by(self, key, key_name=None, key_type=None): groups[group_name].append(row) + if not groups: + return TableSet([self._fork([])], [], key_name=key_name, key_type=key_type) + output = OrderedDict() for group, rows in groups.items(): From 6d683df26260392e251e36926d0a001704283633 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 15:42:13 -0400 Subject: [PATCH 077/163] fix: Table.normalize works with Table.where, closes #691 --- CHANGELOG.rst | 1 + agate/table/normalize.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9070d4aa..3dcf119f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) * feat: :class:`.Max` works with :class:`.TimeDelta`. (#735) * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) +* fix: :meth:`.Table.normalize` works with :meth:`.Table.where`. (#691) * feat: :meth:`.TableSet.group_by` accepts input with no rows. (#703) * fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) diff --git a/agate/table/normalize.py b/agate/table/normalize.py index 7c5b9779..d4e4542f 100644 --- a/agate/table/normalize.py +++ b/agate/table/normalize.py @@ -94,4 +94,4 @@ def normalize(self, key, properties, property_column='property', value_column='v else: new_column_types = key_column_types + list(column_types) - return Table(new_rows, new_column_names, new_column_types, row_names=row_names) + return Table(new_rows, new_column_names, new_column_types) From c6120be72e86775e32c513265df1fa31c569a93b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 15:53:51 -0400 Subject: [PATCH 078/163] docs: Fix typo --- CHANGELOG.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3dcf119f..e443649a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,12 +3,12 @@ * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) -* feat: :class:`.Max` works with :class:`.TimeDelta`. (#735) +* feat: :class:`.Sum` works with :class:`.TimeDelta`. (#735) * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) * fix: :meth:`.Table.normalize` works with :meth:`.Table.where`. (#691) -* feat: :meth:`.TableSet.group_by` accepts input with no rows. (#703) -* fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) +* fix: :meth:`.TableSet.group_by` accepts input with no rows. (#703) +* fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: Ensure files are closed when errors occur. (#734) * build: Make PyICU an optional dependency. From 6ff8d602d5ab822c5be6c91597ba164d82a848ff Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 15:54:12 -0400 Subject: [PATCH 079/163] chore: Add agate.MappedSequence.__repr__, for easier debugging --- agate/mapped_sequence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index 6429c5f3..403528d1 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -83,6 +83,9 @@ def __str__(self): return str(self.__unicode__()) + def __repr__(self): + return self.__str__() + def __getitem__(self, key): """ Retrieve values from this array by index, slice or key. From 8ba352a7b70e079175f64a5066926eebe5d0757f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 15:58:45 -0400 Subject: [PATCH 080/163] feat: Min and Max work with TimeDelta --- CHANGELOG.rst | 2 +- agate/aggregations/max.py | 6 +++--- agate/aggregations/min.py | 6 +++--- tests/test_aggregations.py | 10 ++++++++++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e443649a..da39957d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,7 @@ * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) -* feat: :class:`.Sum` works with :class:`.TimeDelta`. (#735) +* feat: :class:`.Min`, :class:`.Max` and :class:`.Sum` work with :class:`.TimeDelta`. (#735) * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) * fix: :meth:`.Table.normalize` works with :meth:`.Table.where`. (#691) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) diff --git a/agate/aggregations/max.py b/agate/aggregations/max.py index fe83d7b2..e79d1d8e 100644 --- a/agate/aggregations/max.py +++ b/agate/aggregations/max.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from agate.aggregations.base import Aggregation -from agate.data_types import Date, DateTime, Number +from agate.data_types import Date, DateTime, Number, TimeDelta from agate.exceptions import DataTypeError @@ -21,13 +21,13 @@ def __init__(self, column_name): def get_aggregate_data_type(self, table): column = table.columns[self._column_name] - if isinstance(column.data_type, (Number, Date, DateTime)): + if isinstance(column.data_type, (Date, DateTime, Number, TimeDelta)): return column.data_type def validate(self, table): column = table.columns[self._column_name] - if not isinstance(column.data_type, (Number, Date, DateTime)): + if not isinstance(column.data_type, (Date, DateTime, Number, TimeDelta)): raise DataTypeError('Min can only be applied to columns containing DateTime, Date or Number data.') def run(self, table): diff --git a/agate/aggregations/min.py b/agate/aggregations/min.py index f5ab042e..a4161fe9 100644 --- a/agate/aggregations/min.py +++ b/agate/aggregations/min.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from agate.aggregations.base import Aggregation -from agate.data_types import Date, DateTime, Number +from agate.data_types import Date, DateTime, Number, TimeDelta from agate.exceptions import DataTypeError @@ -21,13 +21,13 @@ def __init__(self, column_name): def get_aggregate_data_type(self, table): column = table.columns[self._column_name] - if isinstance(column.data_type, (Number, Date, DateTime)): + if isinstance(column.data_type, (Date, DateTime, Number, TimeDelta)): return column.data_type def validate(self, table): column = table.columns[self._column_name] - if not isinstance(column.data_type, (Number, Date, DateTime)): + if not isinstance(column.data_type, (Date, DateTime, Number, TimeDelta)): raise DataTypeError('Min can only be applied to columns containing DateTime, Date or Number data.') def run(self, table): diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 8f053986..32479cb6 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -205,6 +205,11 @@ def test_min(self): def test_min_all_nulls(self): self.assertIsNone(Min('null').run(self.table)) + def test_min_time_delta(self): + self.assertIsInstance(Min('test').get_aggregate_data_type(self.time_delta_table), TimeDelta) + Min('test').validate(self.time_delta_table) + self.assertEqual(Min('test').run(self.time_delta_table), datetime.timedelta(0, 10)) + def test_max(self): self.assertIsInstance(Max('test').get_aggregate_data_type(self.table), DateTime) Max('test').validate(self.table) @@ -213,6 +218,11 @@ def test_max(self): def test_max_all_nulls(self): self.assertIsNone(Max('null').run(self.table)) + def test_max_time_delta(self): + self.assertIsInstance(Max('test').get_aggregate_data_type(self.time_delta_table), TimeDelta) + Max('test').validate(self.time_delta_table) + self.assertEqual(Max('test').run(self.time_delta_table), datetime.timedelta(0, 20)) + def test_sum(self): self.assertIsInstance(Sum('test').get_aggregate_data_type(self.time_delta_table), TimeDelta) Sum('test').validate(self.time_delta_table) From ddf3b20d5c786243e9860693c4ef02bfc78a6eb2 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 17:38:28 -0400 Subject: [PATCH 081/163] fix: Table.homogenize casts compare_values and default_row, closes #700 --- CHANGELOG.rst | 3 ++- agate/table/homogenize.py | 9 ++++++--- tests/test_table/test_homogenize.py | 8 ++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index da39957d..c42d394d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,9 +3,10 @@ * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) -* feat: :class:`.Min`, :class:`.Max` and :class:`.Sum` work with :class:`.TimeDelta`. (#735) +* feat: :class:`.Min`, :class:`.Max` and :class:`.Sum` (#735) work with :class:`.TimeDelta`. * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) * fix: :meth:`.Table.normalize` works with :meth:`.Table.where`. (#691) +* fix: :meth:`.Table.homogenize` casts ``compare_values`` and ``default_row``. (#700) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) * fix: :meth:`.TableSet.group_by` accepts input with no rows. (#703) * fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) diff --git a/agate/table/homogenize.py b/agate/table/homogenize.py index 0e04ec4e..199f36b8 100644 --- a/agate/table/homogenize.py +++ b/agate/table/homogenize.py @@ -56,21 +56,24 @@ def homogenize(self, key, compare_values, default_row=None): column_values = [self._columns.get(name) for name in key] column_indexes = [self._column_names.index(name) for name in key] + compare_values = [[column_values[i].data_type.cast(v) for i, v in enumerate(values)] for values in compare_values] + column_values = zip(*column_values) differences = list(set(map(tuple, compare_values)) - set(column_values)) for difference in differences: if callable(default_row): - rows.append(Row(default_row(difference), self._column_names)) + new_row = default_row(difference) else: if default_row is not None: new_row = list(default_row) else: - new_row = [None] * (len(self._column_names) - len(key)) + new_row = [None] * len(default_row_types) for i, d in zip(column_indexes, difference): new_row.insert(i, d) - rows.append(Row(new_row, self._column_names)) + new_row = [self._columns[i].data_type.cast(v) for i, v in enumerate(new_row)] + rows.append(Row(new_row, self._column_names)) return self._fork(rows) diff --git a/tests/test_table/test_homogenize.py b/tests/test_table/test_homogenize.py index ecb87a0e..86ea1fd0 100644 --- a/tests/test_table/test_homogenize.py +++ b/tests/test_table/test_homogenize.py @@ -33,6 +33,8 @@ def test_homogenize_column_name(self): (2, 3, 'd') ) + homogenized.print_table() + self.assertColumnNames(homogenized, self.column_names) self.assertColumnTypes(homogenized, [Number, Number, Text]) self.assertRows(homogenized, rows) @@ -48,6 +50,8 @@ def test_homogenize_default_row(self): (2, None, None) ) + homogenized.print_table() + self.assertColumnNames(homogenized, self.column_names) self.assertColumnTypes(homogenized, [Number, Number, Text]) self.assertRows(homogenized, rows) @@ -66,6 +70,8 @@ def column_two(count): (2, 5, 'c') ) + homogenized.print_table() + self.assertColumnNames(homogenized, self.column_names) self.assertColumnTypes(homogenized, [Number, Number, Text]) self.assertRows(homogenized, rows) @@ -87,6 +93,8 @@ def column_two(count): (2, 4, 'c') ) + homogenized.print_table() + self.assertColumnNames(homogenized, self.column_names) self.assertColumnTypes(homogenized, [Number, Number, Text]) self.assertRows(homogenized, rows) From 7a67b857081e2cd40ef2bc7bc6fbb21a63cfb86c Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 17:45:15 -0400 Subject: [PATCH 082/163] docs: Update changelog --- CHANGELOG.rst | 1 + agate/table/from_csv.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f04e02af..b66965ff 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,7 @@ * feat: :meth:`.Table.from_csv` accepts a ``row_limit`` keyword argument. (#740) * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) * feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) +* feat: :class:`.TypeTester` accepts a ``null_values`` keyword argument. (#745) * feat: :class:`.Min`, :class:`.Max` and :class:`.Sum` (#735) work with :class:`.TimeDelta`. * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) * fix: :meth:`.Table.normalize` works with :meth:`.Table.where`. (#691) diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index 4f622a2a..9395c60b 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -40,7 +40,7 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk handle it is assumed you have already opened it with the correct encoding specified. :param row_limit: - Limit how many rows of data will be read + Limit how many rows of data will be read. """ from agate import csv from agate.table import Table From 537ed212f8c858f30ff4d77e8d07d99bfa4f2f9b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 17:50:42 -0400 Subject: [PATCH 083/163] docs: Update changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b66965ff..1974bfea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,7 @@ * fix: :meth:`.Table.homogenize` casts ``compare_values`` and ``default_row``. (#700) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) * fix: :meth:`.TableSet.group_by` accepts input with no rows. (#703) +* fix: :class:`.TypeTester` warns if a column specified by the ``force`` argument is not in the table, instead of raising an error. (#747) * fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: Ensure files are closed when errors occur. (#734) * build: Make PyICU an optional dependency. From 60cdde84b9288c4f04482d72609b69b923be6ffd Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:06:26 -0400 Subject: [PATCH 084/163] fix: Table.Table.homogenize works with basic processing methods, closes #756 --- CHANGELOG.rst | 3 ++- agate/table/homogenize.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1974bfea..3eeb3faf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,9 @@ * feat: :class:`.TypeTester` accepts a ``null_values`` keyword argument. (#745) * feat: :class:`.Min`, :class:`.Max` and :class:`.Sum` (#735) work with :class:`.TimeDelta`. * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) -* fix: :meth:`.Table.normalize` works with :meth:`.Table.where`. (#691) +* fix: :meth:`.Table.normalize` works with basic processing methods. (#691) * fix: :meth:`.Table.homogenize` casts ``compare_values`` and ``default_row``. (#700) +* fix: :meth:`.Table.homogenize` works with basic processing methods. (#756) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) * fix: :meth:`.TableSet.group_by` accepts input with no rows. (#703) * fix: :class:`.TypeTester` warns if a column specified by the ``force`` argument is not in the table, instead of raising an error. (#747) diff --git a/agate/table/homogenize.py b/agate/table/homogenize.py index 199f36b8..e8d5ca9f 100644 --- a/agate/table/homogenize.py +++ b/agate/table/homogenize.py @@ -76,4 +76,5 @@ def homogenize(self, key, compare_values, default_row=None): new_row = [self._columns[i].data_type.cast(v) for i, v in enumerate(new_row)] rows.append(Row(new_row, self._column_names)) - return self._fork(rows) + # Do not copy the row_names, since this function adds rows. + return self._fork(rows, row_names=[]) From 0d7dc4f373e7ed385e03246aa429aefb4f2047c1 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:08:00 -0400 Subject: [PATCH 085/163] chore: flake8 --- agate/table/homogenize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agate/table/homogenize.py b/agate/table/homogenize.py index e8d5ca9f..8a469f64 100644 --- a/agate/table/homogenize.py +++ b/agate/table/homogenize.py @@ -68,7 +68,7 @@ def homogenize(self, key, compare_values, default_row=None): if default_row is not None: new_row = list(default_row) else: - new_row = [None] * len(default_row_types) + new_row = [None] * (len(self._column_names) - len(key)) for i, d in zip(column_indexes, difference): new_row.insert(i, d) From f46c57ea094d1289c6f774537c8f50ca4eb5af2f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:09:34 -0400 Subject: [PATCH 086/163] chore: isort --- agate/type_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agate/type_tester.py b/agate/type_tester.py index 7c5947cb..8f92bbfc 100644 --- a/agate/type_tester.py +++ b/agate/type_tester.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -from copy import copy import warnings +from copy import copy from agate.data_types.base import DEFAULT_NULL_VALUES from agate.data_types.boolean import Boolean From 4bd0d227a38ee84797bf4463b140b782f80e4ce7 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:11:16 -0400 Subject: [PATCH 087/163] ci: Attempt testing example on Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55e4361e..1ab525c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,6 @@ jobs: - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate # UnicodeDecodeError on print_bars - - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' + - if: matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' run: python example.py - run: python charts.py From 9f0157d183f7c4311ecb6f48ad969b48f48e62c4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:13:39 -0400 Subject: [PATCH 088/163] ci: Confirmed testing example fails on Windows and Python 3.9 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ab525c0..55e4361e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,6 @@ jobs: - run: pip install .[test] - run: nosetests --with-coverage --cover-package=agate # UnicodeDecodeError on print_bars - - if: matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' + - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' run: python example.py - run: python charts.py From 79b17b4aa24897542a6a8718d4f6dbac640d5dd6 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:27:32 -0400 Subject: [PATCH 089/163] test: Remove empty setUp methods --- tests/test_py2.py | 3 --- tests/test_py3.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/test_py2.py b/tests/test_py2.py index d1e384e8..817120c6 100644 --- a/tests/test_py2.py +++ b/tests/test_py2.py @@ -350,9 +350,6 @@ def test_writer_alias(self): @unittest.skipIf(six.PY3, "Not supported in Python 3.") class TestSniffer(unittest.TestCase): - def setUp(self): - pass - def test_sniffer(self): with open('examples/test.csv') as f: contents = f.read() diff --git a/tests/test_py3.py b/tests/test_py3.py index 8d5ab750..078860d2 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -245,9 +245,6 @@ def test_writerows(self): @unittest.skipIf(six.PY2, "Not supported in Python 2.") class TestSniffer(unittest.TestCase): - def setUp(self): - pass - def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() From 07639829c0677f0d9c24807970957601d0cc118d Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 18:28:21 -0400 Subject: [PATCH 090/163] test: Set maxDiff on TestSniffer --- CHANGELOG.rst | 4 ++-- tests/test_py2.py | 2 ++ tests/test_py3.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3eeb3faf..c8d72916 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,5 @@ -1.6.3 - Unreleased ------------------- +1.6.3 - July 14, 2021 +--------------------- * feat: :meth:`.Table.from_csv` accepts a ``row_limit`` keyword argument. (#740) * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) diff --git a/tests/test_py2.py b/tests/test_py2.py index 817120c6..6eaa8c48 100644 --- a/tests/test_py2.py +++ b/tests/test_py2.py @@ -350,6 +350,8 @@ def test_writer_alias(self): @unittest.skipIf(six.PY3, "Not supported in Python 3.") class TestSniffer(unittest.TestCase): + maxDiff = None + def test_sniffer(self): with open('examples/test.csv') as f: contents = f.read() diff --git a/tests/test_py3.py b/tests/test_py3.py index 078860d2..f9f2e473 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -245,6 +245,8 @@ def test_writerows(self): @unittest.skipIf(six.PY2, "Not supported in Python 2.") class TestSniffer(unittest.TestCase): + maxDiff = None + def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() From 79510f4b941c1f08ae7c0c4c05d61a87304954e6 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 19:12:06 -0400 Subject: [PATCH 091/163] chore: Use specific exception to avoid shadowing bug --- agate/csv_py2.py | 2 +- agate/csv_py3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/agate/csv_py2.py b/agate/csv_py2.py index e84c627f..c4148461 100644 --- a/agate/csv_py2.py +++ b/agate/csv_py2.py @@ -250,7 +250,7 @@ def sniff(self, sample): """ try: dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except Exception: + except csv.Error: dialect = None return dialect diff --git a/agate/csv_py3.py b/agate/csv_py3.py index e7183182..08101011 100644 --- a/agate/csv_py3.py +++ b/agate/csv_py3.py @@ -151,7 +151,7 @@ def sniff(self, sample): """ try: dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except Exception: + except csv.Error: dialect = None return dialect From b15ae828cd4041298fa13cf14371166aa8a23171 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jul 2021 19:21:14 -0400 Subject: [PATCH 092/163] test: Add explicit message since maxDiff seems to be a no-op --- tests/test_py3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_py3.py b/tests/test_py3.py index f9f2e473..2a81f143 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -250,4 +250,5 @@ class TestSniffer(unittest.TestCase): def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() - self.assertEqual(csv_py3.Sniffer().sniff(contents).__dict__, csv.Sniffer().sniff(contents).__dict__) + self.assertEqual(csv_py3.Sniffer().sniff(contents).__dict__, csv.Sniffer().sniff(contents).__dict__, + repr(csv.Sniffer().sniff(contents).__dict__)) From 261a907801c12f51ade33daeb3e21c23d54ee7f6 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:34:43 -0400 Subject: [PATCH 093/163] chore: flake8 E501 --- agate/computations/change.py | 3 ++- agate/data_types/boolean.py | 3 ++- agate/data_types/number.py | 6 ++++-- agate/table/__init__.py | 10 +++++++--- agate/table/compute.py | 4 +++- agate/table/denormalize.py | 3 ++- agate/table/from_csv.py | 3 ++- agate/table/from_fixed.py | 3 ++- agate/table/from_json.py | 8 +++++--- agate/table/join.py | 3 ++- agate/table/pivot.py | 3 ++- agate/table/print_bars.py | 3 ++- agate/table/print_table.py | 3 ++- setup.cfg | 1 - tests/test_data_types.py | 15 +++++++++++--- tests/test_table/test_bins.py | 30 ++++++++++++++++++++++------ tests/test_table/test_denormalize.py | 3 ++- tests/test_tableset/__init__.py | 6 +++++- 18 files changed, 80 insertions(+), 30 deletions(-) diff --git a/agate/computations/change.py b/agate/computations/change.py index 4925f67b..c210e9dc 100644 --- a/agate/computations/change.py +++ b/agate/computations/change.py @@ -49,7 +49,8 @@ def validate(self, table): return - raise DataTypeError('Change before and after columns must both contain data that is one of: Number, Date, DateTime or TimeDelta.') + raise DataTypeError('Change before and after columns must both contain data that is one of: ' + 'Number, Date, DateTime or TimeDelta.') def run(self, table): new_column = [] diff --git a/agate/data_types/boolean.py b/agate/data_types/boolean.py index 6b4908dc..05791156 100644 --- a/agate/data_types/boolean.py +++ b/agate/data_types/boolean.py @@ -29,7 +29,8 @@ class Boolean(DataType): :param false_values: A sequence of values which should be cast to :code:`False` when encountered with this type. """ - def __init__(self, true_values=DEFAULT_TRUE_VALUES, false_values=DEFAULT_FALSE_VALUES, null_values=DEFAULT_NULL_VALUES): + def __init__(self, true_values=DEFAULT_TRUE_VALUES, false_values=DEFAULT_FALSE_VALUES, + null_values=DEFAULT_NULL_VALUES): super(Boolean, self).__init__(null_values=null_values) self.true_values = true_values diff --git a/agate/data_types/number.py b/agate/data_types/number.py index 56bd3c31..c875cb3a 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -15,7 +15,8 @@ from agate.exceptions import CastError #: A list of currency symbols sourced from `Xe `_. -DEFAULT_CURRENCY_SYMBOLS = [u'؋', u'$', u'ƒ', u'៛', u'¥', u'₡', u'₱', u'£', u'€', u'¢', u'﷼', u'₪', u'₩', u'₭', u'₮', u'₦', u'฿', u'₤', u'₫'] +DEFAULT_CURRENCY_SYMBOLS = [u'؋', u'$', u'ƒ', u'៛', u'¥', u'₡', u'₱', u'£', u'€', u'¢', u'﷼', u'₪', u'₩', u'₭', u'₮', + u'₦', u'฿', u'₤', u'₫'] POSITIVE = Decimal('1') NEGATIVE = Decimal('-1') @@ -37,7 +38,8 @@ class Number(DataType): :param currency_symbols: A sequence of currency symbols to strip from numbers. """ - def __init__(self, locale='en_US', group_symbol=None, decimal_symbol=None, currency_symbols=DEFAULT_CURRENCY_SYMBOLS, **kwargs): + def __init__(self, locale='en_US', group_symbol=None, decimal_symbol=None, + currency_symbols=DEFAULT_CURRENCY_SYMBOLS, **kwargs): super(Number, self).__init__(**kwargs) self.locale = Locale.parse(locale) diff --git a/agate/table/__init__.py b/agate/table/__init__.py index fba388dc..3affe053 100644 --- a/agate/table/__init__.py +++ b/agate/table/__init__.py @@ -77,14 +77,16 @@ class Table(object): """ def __init__(self, rows, column_names=None, column_types=None, row_names=None, _is_fork=False): if isinstance(rows, six.string_types): - raise ValueError('When created directly, the first argument to Table must be a sequence of rows. Did you want agate.Table.from_csv?') + raise ValueError('When created directly, the first argument to Table must be a sequence of rows. ' + 'Did you want agate.Table.from_csv?') # Validate column names if column_names: self._column_names = utils.deduplicate(column_names, column_names=True) elif rows: self._column_names = tuple(utils.letter_name(i) for i in range(len(rows[0]))) - warnings.warn('Column names not specified. "%s" will be used as names.' % str(self._column_names), RuntimeWarning, stacklevel=2) + warnings.warn('Column names not specified. "%s" will be used as names.' % str(self._column_names), + RuntimeWarning, stacklevel=2) else: self._column_names = tuple() @@ -120,7 +122,9 @@ def __init__(self, rows, column_names=None, column_types=None, row_names=None, _ len_row = len(row) if len_row > len_column_names: - raise ValueError('Row %i has %i values, but Table only has %i columns.' % (i, len_row, len_column_names)) + raise ValueError( + 'Row %i has %i values, but Table only has %i columns.' % (i, len_row, len_column_names) + ) elif len(row) < len_column_names: row = chain(row, [None] * (len_column_names - len_row)) diff --git a/agate/table/compute.py b/agate/table/compute.py index 887fe88f..6260ecf5 100644 --- a/agate/table/compute.py +++ b/agate/table/compute.py @@ -29,7 +29,9 @@ def compute(self, computations, replace=False): if new_column_name in column_names: if not replace: - raise ValueError('New column name "%s" already exists. Specify replace=True to replace with computed data.') + raise ValueError( + 'New column name "%s" already exists. Specify replace=True to replace with computed data.' + ) i = column_names.index(new_column_name) column_types[i] = new_column_type diff --git a/agate/table/denormalize.py b/agate/table/denormalize.py index d1aa4231..03e07953 100644 --- a/agate/table/denormalize.py +++ b/agate/table/denormalize.py @@ -16,7 +16,8 @@ from agate.type_tester import TypeTester -def denormalize(self, key=None, property_column='property', value_column='value', default_value=utils.default, column_types=None): +def denormalize(self, key=None, property_column='property', value_column='value', default_value=utils.default, + column_types=None): """ Create a new table with row values converted into columns. diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index 9395c60b..81daa72b 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -7,7 +7,8 @@ @classmethod -def from_csv(cls, path, column_names=None, column_types=None, row_names=None, skip_lines=0, header=True, sniff_limit=0, encoding='utf-8', row_limit=None, **kwargs): +def from_csv(cls, path, column_names=None, column_types=None, row_names=None, skip_lines=0, header=True, sniff_limit=0, + encoding='utf-8', row_limit=None, **kwargs): """ Create a new table from a CSV. diff --git a/agate/table/from_fixed.py b/agate/table/from_fixed.py index de88fd3b..c0fa4da4 100644 --- a/agate/table/from_fixed.py +++ b/agate/table/from_fixed.py @@ -6,7 +6,8 @@ @classmethod -def from_fixed(cls, path, schema_path, column_names=utils.default, column_types=None, row_names=None, encoding='utf-8', schema_encoding='utf-8'): +def from_fixed(cls, path, schema_path, column_names=utils.default, column_types=None, row_names=None, encoding='utf-8', + schema_encoding='utf-8'): """ Create a new table from a fixed-width file and a CSV schema. diff --git a/agate/table/from_json.py b/agate/table/from_json.py index 78461ef6..2b673f1e 100644 --- a/agate/table/from_json.py +++ b/agate/table/from_json.py @@ -33,8 +33,8 @@ def from_json(cls, path, row_names=None, key=None, newline=False, column_types=N :param encoding: According to RFC4627, JSON text shall be encoded in Unicode; the default encoding is UTF-8. You can override this by using any encoding supported by your Python's open() function - if :code:`path` is a filepath. If passing in a file handle, it is assumed you have already opened it with the correct - encoding specified. + if :code:`path` is a filepath. If passing in a file handle, it is assumed you have already opened it with the + correct encoding specified. """ from agate.table import Table @@ -67,7 +67,9 @@ def from_json(cls, path, row_names=None, key=None, newline=False, column_types=N if isinstance(js, dict): if not key: - raise TypeError('When converting a JSON document with a top-level dictionary element, a key must be specified.') + raise TypeError( + 'When converting a JSON document with a top-level dictionary element, a key must be specified.' + ) js = js[key] diff --git a/agate/table/join.py b/agate/table/join.py index e30d820d..9b6ea37f 100644 --- a/agate/table/join.py +++ b/agate/table/join.py @@ -5,7 +5,8 @@ from agate.rows import Row -def join(self, right_table, left_key=None, right_key=None, inner=False, full_outer=False, require_match=False, columns=None): +def join(self, right_table, left_key=None, right_key=None, inner=False, full_outer=False, require_match=False, + columns=None): """ Create a new table by joining two table's on common values. This method implements most varieties of SQL join, in addition to some unique features. diff --git a/agate/table/pivot.py b/agate/table/pivot.py index 9db82aaf..9b259cb1 100644 --- a/agate/table/pivot.py +++ b/agate/table/pivot.py @@ -124,7 +124,8 @@ def apply_computation(table): column_types = [column_type] * pivot_count - table = table.denormalize(key, pivot, computation_name or aggregation_name, default_value=default_value, column_types=column_types) + table = table.denormalize(key, pivot, computation_name or aggregation_name, default_value=default_value, + column_types=column_types) else: table = groups.aggregate([ (aggregation_name, aggregation) diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index cddbccc3..a7aadaf3 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -20,7 +20,8 @@ from agate.exceptions import DataTypeError -def print_bars(self, label_column_name='group', value_column_name='Count', domain=None, width=120, output=sys.stdout, printable=False): +def print_bars(self, label_column_name='group', value_column_name='Count', domain=None, width=120, output=sys.stdout, + printable=False): """ Print a text-based bar chart based on this table. diff --git a/agate/table/print_table.py b/agate/table/print_table.py index f3fa4712..d20436bd 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -11,7 +11,8 @@ from agate.data_types import Number, Text -def print_table(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_width=20, locale=None, max_precision=3): +def print_table(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_width=20, locale=None, + max_precision=3): """ Print a text-based view of the data in this table. diff --git a/setup.cfg b/setup.cfg index 378d4a7d..31cf81cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,5 @@ [flake8] max-line-length = 119 -extend-ignore = E501 per-file-ignores = # imported but unused, unable to detect undefined names agate/__init__.py: F401,F403 diff --git a/tests/test_data_types.py b/tests/test_data_types.py index cd574b23..e284c741 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -149,7 +149,10 @@ def test_test(self): def test_cast(self): values = (2, 1, None, Decimal('2.7'), 'n/a', '2.7', '200,000,000') casted = tuple(self.type.cast(v) for v in values) - self.assertSequenceEqual(casted, (Decimal('2'), Decimal('1'), None, Decimal('2.7'), None, Decimal('2.7'), Decimal('200000000'))) + self.assertSequenceEqual( + casted, + (Decimal('2'), Decimal('1'), None, Decimal('2.7'), None, Decimal('2.7'), Decimal('200000000')) + ) @unittest.skipIf(six.PY3, 'Not supported in Python 3.') def test_cast_long(self): @@ -164,12 +167,18 @@ def test_boolean_cast(self): def test_currency_cast(self): values = ('$2.70', '-$0.70', u'€14', u'50¢', u'-75¢', u'-$1,287') casted = tuple(self.type.cast(v) for v in values) - self.assertSequenceEqual(casted, (Decimal('2.7'), Decimal('-0.7'), Decimal('14'), Decimal('50'), Decimal('-75'), Decimal('-1287'))) + self.assertSequenceEqual( + casted, + (Decimal('2.7'), Decimal('-0.7'), Decimal('14'), Decimal('50'), Decimal('-75'), Decimal('-1287')) + ) def test_cast_locale(self): values = (2, 1, None, Decimal('2.7'), 'n/a', '2,7', '200.000.000') casted = tuple(Number(locale='de_DE.UTF-8').cast(v) for v in values) - self.assertSequenceEqual(casted, (Decimal('2'), Decimal('1'), None, Decimal('2.7'), None, Decimal('2.7'), Decimal('200000000'))) + self.assertSequenceEqual( + casted, + (Decimal('2'), Decimal('1'), None, Decimal('2.7'), None, Decimal('2.7'), Decimal('200000000')) + ) def test_cast_text(self): with self.assertRaises(CastError): diff --git a/tests/test_table/test_bins.py b/tests/test_table/test_bins.py index 96795630..2cf6d008 100644 --- a/tests/test_table/test_bins.py +++ b/tests/test_table/test_bins.py @@ -112,9 +112,18 @@ def test_bins_decimals(self): self.assertColumnNames(new_table, ['number', 'Count']) self.assertColumnTypes(new_table, [Text, Number]) - self.assertSequenceEqual(new_table.rows[0], [u'[0' + get_decimal_symbol() + u'0 - 0' + get_decimal_symbol() + u'1)', 10]) - self.assertSequenceEqual(new_table.rows[3], [u'[0' + get_decimal_symbol() + u'3 - 0' + get_decimal_symbol() + u'4)', 10]) - self.assertSequenceEqual(new_table.rows[9], [u'[0' + get_decimal_symbol() + u'9 - 1' + get_decimal_symbol() + u'0]', 10]) + self.assertSequenceEqual( + new_table.rows[0], + [u'[0' + get_decimal_symbol() + u'0 - 0' + get_decimal_symbol() + u'1)', 10] + ) + self.assertSequenceEqual( + new_table.rows[3], + [u'[0' + get_decimal_symbol() + u'3 - 0' + get_decimal_symbol() + u'4)', 10] + ) + self.assertSequenceEqual( + new_table.rows[9], + [u'[0' + get_decimal_symbol() + u'9 - 1' + get_decimal_symbol() + u'0]', 10] + ) def test_bins_nulls(self): rows = [] @@ -129,7 +138,16 @@ def test_bins_nulls(self): self.assertColumnNames(new_table, ['number', 'Count']) self.assertColumnTypes(new_table, [Text, Number]) - self.assertSequenceEqual(new_table.rows[0], [u'[0' + get_decimal_symbol() + u'0 - 0' + get_decimal_symbol() + u'1)', 10]) - self.assertSequenceEqual(new_table.rows[3], [u'[0' + get_decimal_symbol() + u'3 - 0' + get_decimal_symbol() + u'4)', 10]) - self.assertSequenceEqual(new_table.rows[9], [u'[0' + get_decimal_symbol() + u'9 - 1' + get_decimal_symbol() + u'0]', 10]) + self.assertSequenceEqual( + new_table.rows[0], + [u'[0' + get_decimal_symbol() + u'0 - 0' + get_decimal_symbol() + u'1)', 10] + ) + self.assertSequenceEqual( + new_table.rows[3], + [u'[0' + get_decimal_symbol() + u'3 - 0' + get_decimal_symbol() + u'4)', 10] + ) + self.assertSequenceEqual( + new_table.rows[9], + [u'[0' + get_decimal_symbol() + u'9 - 1' + get_decimal_symbol() + u'0]', 10] + ) self.assertSequenceEqual(new_table.rows[10], [None, 1]) diff --git a/tests/test_table/test_denormalize.py b/tests/test_table/test_denormalize.py index 9eb52598..d85f878c 100644 --- a/tests/test_table/test_denormalize.py +++ b/tests/test_table/test_denormalize.py @@ -98,7 +98,8 @@ def test_denormalize_column_types(self): def test_denormalize_column_type_tester(self): table = Table(self.rows, self.column_names, self.column_types) - normalized_table = table.denormalize(None, 'property', 'value', column_types=TypeTester(force={'gender': Text()})) + type_tester = TypeTester(force={'gender': Text()}) + normalized_table = table.denormalize(None, 'property', 'value', column_types=type_tester) # NB: value has been overwritten normal_rows = ( diff --git a/tests/test_tableset/__init__.py b/tests/test_tableset/__init__.py index a1a8c54c..508bfcb2 100644 --- a/tests/test_tableset/__init__.py +++ b/tests/test_tableset/__init__.py @@ -147,7 +147,11 @@ def test_from_json_file(self): tableset3 = TableSet.from_json(filelike) self.assertSequenceEqual(tableset1.column_names, tableset2.column_names, tableset3.column_names) - self.assertSequenceEqual([type(t) for t in tableset1.column_types], [type(t) for t in tableset2.column_types], [type(t) for t in tableset3.column_types]) + self.assertSequenceEqual( + [type(t) for t in tableset1.column_types], + [type(t) for t in tableset2.column_types], + [type(t) for t in tableset3.column_types] + ) self.assertEqual(len(tableset1), len(tableset2), len(tableset3)) From c6222f3ceb782c2c2315edc12c79bc366688a5dd Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:39:06 -0400 Subject: [PATCH 094/163] feat: Warn on error while sniffing CSV dialect --- agate/csv_py3.py | 4 +++- tests/test_py3.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/agate/csv_py3.py b/agate/csv_py3.py index 08101011..94354882 100644 --- a/agate/csv_py3.py +++ b/agate/csv_py3.py @@ -5,6 +5,7 @@ """ import csv +import warnings import six @@ -151,7 +152,8 @@ def sniff(self, sample): """ try: dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except csv.Error: + except csv.Error as e: + warnings.warn('Error sniffing CSV dialect: %s' % e, RuntimeWarning, stacklevel=2) dialect = None return dialect diff --git a/tests/test_py3.py b/tests/test_py3.py index 2a81f143..461692d1 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -250,5 +250,6 @@ class TestSniffer(unittest.TestCase): def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() - self.assertEqual(csv_py3.Sniffer().sniff(contents).__dict__, csv.Sniffer().sniff(contents).__dict__, - repr(csv.Sniffer().sniff(contents).__dict__)) + actual = csv_py3.Sniffer().sniff(contents).__dict__ + expected = csv.Sniffer().sniff(contents).__dict__ + self.assertEqual(actual, expected, '%r != %r' % (actual, expected)) From b4f7b712b4d24a7d1929e1a779bee595182a3bac Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:44:12 -0400 Subject: [PATCH 095/163] test: Expand test failure message for debugging --- CHANGELOG.rst | 1 + agate/csv_py2.py | 3 ++- tests/test_py3.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c8d72916..7165984e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,7 @@ * feat: :class:`.TypeTester` accepts a ``null_values`` keyword argument. (#745) * feat: :class:`.Min`, :class:`.Max` and :class:`.Sum` (#735) work with :class:`.TimeDelta`. * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) +* feat: :class:`.csv.Sniffer` warns on error while sniffing CSV dialect. * fix: :meth:`.Table.normalize` works with basic processing methods. (#691) * fix: :meth:`.Table.homogenize` casts ``compare_values`` and ``default_row``. (#700) * fix: :meth:`.Table.homogenize` works with basic processing methods. (#756) diff --git a/agate/csv_py2.py b/agate/csv_py2.py index c4148461..5c8bcd39 100644 --- a/agate/csv_py2.py +++ b/agate/csv_py2.py @@ -250,7 +250,8 @@ def sniff(self, sample): """ try: dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except csv.Error: + except csv.Error as e: + warnings.warn('Error sniffing CSV dialect: %s' % e, RuntimeWarning, stacklevel=2) dialect = None return dialect diff --git a/tests/test_py3.py b/tests/test_py3.py index 461692d1..e4ebd22d 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -252,4 +252,5 @@ def test_sniffer(self): contents = f.read() actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ - self.assertEqual(actual, expected, '%r != %r' % (actual, expected)) + alternate = csv.Sniffer().sniff(contents, csv_py3.POSSIBLE_DELIMITERS).__dict__ + self.assertEqual(actual, expected, '%r != %r (%r)' % (actual, expected, alternate)) From daa04be65c8ec12bf13d330cf6ed2d2f2291de10 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:45:53 -0400 Subject: [PATCH 096/163] chore: flake8 --- agate/csv_py2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/agate/csv_py2.py b/agate/csv_py2.py index 5c8bcd39..ea30f77f 100644 --- a/agate/csv_py2.py +++ b/agate/csv_py2.py @@ -6,6 +6,7 @@ import codecs import csv +import warnings import six From a24ea5f030b9517ba826c1805d3a9a61a2241c71 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:02:36 -0400 Subject: [PATCH 097/163] test: Skip test that inexplicably fails on Linux on Python 3.6 --- tests/test_py2.py | 2 -- tests/test_py3.py | 13 ++++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_py2.py b/tests/test_py2.py index 6eaa8c48..817120c6 100644 --- a/tests/test_py2.py +++ b/tests/test_py2.py @@ -350,8 +350,6 @@ def test_writer_alias(self): @unittest.skipIf(six.PY3, "Not supported in Python 3.") class TestSniffer(unittest.TestCase): - maxDiff = None - def test_sniffer(self): with open('examples/test.csv') as f: contents = f.read() diff --git a/tests/test_py3.py b/tests/test_py3.py index e4ebd22d..a5a0fc7b 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import csv +import platform import os try: @@ -243,14 +244,16 @@ def test_writerows(self): self.assertEqual(result, 'line_number,a,b,c\n1,1,2,☃\n') -@unittest.skipIf(six.PY2, "Not supported in Python 2.") +@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and sys.version_info == (3, 6), + "Not supported in Python 2. Test fails inexplicably on Linux in Python 3.6.") class TestSniffer(unittest.TestCase): - maxDiff = None - def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() + # Inexplicably, `direct` succeeds but `actual` succeeds. + direct = csv.Sniffer().sniff(contents, csv_py3.POSSIBLE_DELIMITERS).__dict__ actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ - alternate = csv.Sniffer().sniff(contents, csv_py3.POSSIBLE_DELIMITERS).__dict__ - self.assertEqual(actual, expected, '%r != %r (%r)' % (actual, expected, alternate)) + + self.assertEqual(direct, expected, '%r != %r (%r)' % (direct, expected)) + self.assertEqual(actual, expected, '%r != %r (%r)' % (actual, expected)) From 01cdc123ebfa50c74aa5b772116c9fb79e2a559b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:03:11 -0400 Subject: [PATCH 098/163] test: Fix sys.version_info comparison --- tests/test_py3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_py3.py b/tests/test_py3.py index a5a0fc7b..773e0e2c 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -244,7 +244,7 @@ def test_writerows(self): self.assertEqual(result, 'line_number,a,b,c\n1,1,2,☃\n') -@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and sys.version_info == (3, 6), +@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and sys.version_info[:2] == (3, 6), "Not supported in Python 2. Test fails inexplicably on Linux in Python 3.6.") class TestSniffer(unittest.TestCase): def test_sniffer(self): From 436444c0d493fef3177e5d9e771f7443cca97663 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:05:25 -0400 Subject: [PATCH 099/163] chore: Remove outdated comment --- agate/table/print_table.py | 2 -- tests/test_py3.py | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/agate/table/print_table.py b/agate/table/print_table.py index d20436bd..101ec5b3 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -140,8 +140,6 @@ def write_row(formatted_row): write('%s%s%s' % (v_line, text, v_line)) - # Dashes span each width with '+' character at intersection of - # horizontal and vertical dividers. divider = '%(v_line)s %(columns)s %(v_line)s' % { 'v_line': v_line, 'columns': ' | '.join(h_line * w for w in widths) diff --git a/tests/test_py3.py b/tests/test_py3.py index 773e0e2c..8ec0d627 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -4,6 +4,7 @@ import csv import platform import os +import sys try: import unittest2 as unittest @@ -255,5 +256,5 @@ def test_sniffer(self): actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ - self.assertEqual(direct, expected, '%r != %r (%r)' % (direct, expected)) - self.assertEqual(actual, expected, '%r != %r (%r)' % (actual, expected)) + self.assertEqual(direct, expected, '%r != %r' % (direct, expected)) + self.assertEqual(actual, expected, '%r != %r' % (actual, expected)) From 4847a18a090993c8ad13c06df05e77bb14df8136 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:07:14 -0400 Subject: [PATCH 100/163] chore: isort, add pre-commit configuration to avoid such issues --- .pre-commit-config.yaml | 9 +++++++++ tests/test_py3.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..6ab68d6e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: + - repo: https://github.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/isort + rev: 5.8.0 + hooks: + - id: isort diff --git a/tests/test_py3.py b/tests/test_py3.py index 8ec0d627..a57b6c5b 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- import csv -import platform import os +import platform import sys try: From 091957bc89862a0d0b877581872d6fb6d178e088 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:14:19 -0400 Subject: [PATCH 101/163] chore: check-manifest --- .pre-commit-config.yaml | 4 ++++ MANIFEST.in | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ab68d6e..bc671142 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,3 +7,7 @@ repos: rev: 5.8.0 hooks: - id: isort + - repo: https://github.com/mgedmin/check-manifest + rev: "0.46" + hooks: + - id: check-manifest diff --git a/MANIFEST.in b/MANIFEST.in index 4a480615..d88af1d9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include *.ipynb include *.py include *.rst +include *.yaml include COPYING recursive-include benchmarks *.py recursive-include docs *.py @@ -11,3 +12,4 @@ recursive-include examples *.csv recursive-include examples *.json recursive-include examples testfixed recursive-include tests *.py +global-exclude *.pyc From ca66f716fc467e9de2f330999e0b30efaada8133 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:17:47 -0400 Subject: [PATCH 102/163] chore: Make charts.py executable --- MANIFEST.in | 2 +- charts.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 charts.py diff --git a/MANIFEST.in b/MANIFEST.in index d88af1d9..bddc21f1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ include *.ipynb include *.py include *.rst -include *.yaml include COPYING recursive-include benchmarks *.py recursive-include docs *.py @@ -12,4 +11,5 @@ recursive-include examples *.csv recursive-include examples *.json recursive-include examples testfixed recursive-include tests *.py +exclude .pre-commit-config.yaml global-exclude *.pyc diff --git a/charts.py b/charts.py old mode 100644 new mode 100755 From 9f3808be5ecd1566be43b537d606f1192a007c19 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:20:44 -0400 Subject: [PATCH 103/163] test: Skip test that inexplicably fails intermittently on Linux on Python 3 --- tests/test_py3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_py3.py b/tests/test_py3.py index a57b6c5b..bc591841 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -245,13 +245,13 @@ def test_writerows(self): self.assertEqual(result, 'line_number,a,b,c\n1,1,2,☃\n') -@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and sys.version_info[:2] == (3, 6), - "Not supported in Python 2. Test fails inexplicably on Linux in Python 3.6.") +@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and sys.version_info > (3,), + "Not supported in Python 2. Test inexplicably fails intermittently on Linux and Python 3.") class TestSniffer(unittest.TestCase): def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() - # Inexplicably, `direct` succeeds but `actual` succeeds. + # Inexplicably, `direct` succeeds but `actual` succeeds. Observed on Python 3.6 and 3.8. direct = csv.Sniffer().sniff(contents, csv_py3.POSSIBLE_DELIMITERS).__dict__ actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ From dc0684a082286bd11c50a32045a9624fb6b3a60f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:32:08 -0400 Subject: [PATCH 104/163] build: Iterate the version number --- CHANGELOG.rst | 8 ++++---- docs/conf.py | 2 +- setup.py | 2 +- tests/test_py3.py | 4 +++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7165984e..88100649 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,16 +1,16 @@ -1.6.3 - July 14, 2021 +1.6.3 - July 15, 2021 --------------------- * feat: :meth:`.Table.from_csv` accepts a ``row_limit`` keyword argument. (#740) * feat: :meth:`.Table.from_json` accepts an ``encoding`` keyword argument. (#734) -* feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument. (#753) -* feat: :class:`.TypeTester` accepts a ``null_values`` keyword argument. (#745) +* feat: :meth:`.Table.print_html` accepts a ``max_precision`` keyword argument, like :meth:`.Table.print_table`. (#753) +* feat: :class:`.TypeTester` accepts a ``null_values`` keyword argument, like individual data types. (#745) * feat: :class:`.Min`, :class:`.Max` and :class:`.Sum` (#735) work with :class:`.TimeDelta`. * feat: :class:`.FieldSizeLimitError` includes the line number in the error message. (#681) * feat: :class:`.csv.Sniffer` warns on error while sniffing CSV dialect. * fix: :meth:`.Table.normalize` works with basic processing methods. (#691) -* fix: :meth:`.Table.homogenize` casts ``compare_values`` and ``default_row``. (#700) * fix: :meth:`.Table.homogenize` works with basic processing methods. (#756) +* fix: :meth:`.Table.homogenize` casts ``compare_values`` and ``default_row``. (#700) * fix: :meth:`.Table.homogenize` accepts tuples. (#710) * fix: :meth:`.TableSet.group_by` accepts input with no rows. (#703) * fix: :class:`.TypeTester` warns if a column specified by the ``force`` argument is not in the table, instead of raising an error. (#747) diff --git a/docs/conf.py b/docs/conf.py index 1f17df4a..4153a3c8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ # Metadata project = u'agate' copyright = u'2017, Christopher Groskopf' -version = '1.6.2' +version = '1.6.3' release = version exclude_patterns = ['_build'] diff --git a/setup.py b/setup.py index d992880a..39a28ea5 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='agate', - version='1.6.2', + version='1.6.3', description='A data analysis library that is optimized for humans instead of machines.', long_description=long_description, long_description_content_type='text/x-rst', diff --git a/tests/test_py3.py b/tests/test_py3.py index bc591841..b2e4d515 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -251,7 +251,9 @@ class TestSniffer(unittest.TestCase): def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() - # Inexplicably, `direct` succeeds but `actual` succeeds. Observed on Python 3.6 and 3.8. + # Failure observed on Python 3.6 and 3.8. The left-hand side in these cases is an empty dictionary. + # 3.6 https://github.com/wireservice/agate/runs/3078380985 + # 3.8 https://github.com/wireservice/agate/runs/3078633299 direct = csv.Sniffer().sniff(contents, csv_py3.POSSIBLE_DELIMITERS).__dict__ actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ From 06560ea8644ee238a2671510a8a76a57bf9694bb Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:37:18 -0400 Subject: [PATCH 105/163] build: Add Python 2.7 classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 39a28ea5..fde9583b 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', From 9b12d4bcc75bf3788e0774e23188f4409c3e7519 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 15 Jul 2021 13:22:49 -0400 Subject: [PATCH 106/163] test: Use six instead of six.version_info --- tests/test_py3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_py3.py b/tests/test_py3.py index b2e4d515..e7055154 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -4,7 +4,6 @@ import csv import os import platform -import sys try: import unittest2 as unittest @@ -245,7 +244,7 @@ def test_writerows(self): self.assertEqual(result, 'line_number,a,b,c\n1,1,2,☃\n') -@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and sys.version_info > (3,), +@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and six.PY3, "Not supported in Python 2. Test inexplicably fails intermittently on Linux and Python 3.") class TestSniffer(unittest.TestCase): def test_sniffer(self): From bfb07a032abbaa901d37f3b7a9958bea675ec4c5 Mon Sep 17 00:00:00 2001 From: Andrii Oriekhov Date: Wed, 2 Mar 2022 11:28:32 +0200 Subject: [PATCH 107/163] add GitHub URL for PyPi --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index fde9583b..af852d69 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,9 @@ author='Christopher Groskopf', author_email='chrisgroskopf@gmail.com', url='http://agate.readthedocs.org/', + project_urls={ + 'Source': 'https://github.com/wireservice/agate', + }, license='MIT', classifiers=[ 'Development Status :: 5 - Production/Stable', From ac89331c87b06e408d10f6a6d8b3c789eb4fc09c Mon Sep 17 00:00:00 2001 From: Castor Fu <6226927+castorf@users.noreply.github.com> Date: Sun, 6 Nov 2022 16:14:49 -0800 Subject: [PATCH 108/163] lowercase null_values for data_types. --- agate/data_types/base.py | 2 +- tests/test_data_types.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/agate/data_types/base.py b/agate/data_types/base.py index de7eeb5a..d15c5afc 100644 --- a/agate/data_types/base.py +++ b/agate/data_types/base.py @@ -16,7 +16,7 @@ class DataType(object): # pragma: no cover :code:`None` when encountered by this data type. """ def __init__(self, null_values=DEFAULT_NULL_VALUES): - self.null_values = null_values + self.null_values = [v.lower() for v in null_values] def test(self, d): """ diff --git a/tests/test_data_types.py b/tests/test_data_types.py index e284c741..cf8c32a2 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -63,6 +63,10 @@ def test_no_cast_nulls(self): casted = tuple(t.cast(v) for v in values) self.assertSequenceEqual(casted, ('', 'N/A', None)) + def test_null_values(self): + t = Text(null_values=['Bad Value']) + self.assertEqual(t.cast('Bad Value'), None) + class TestBoolean(unittest.TestCase): def setUp(self): From e4fe0ea5c1ffd7b6ac86764018b70d81424f8a5b Mon Sep 17 00:00:00 2001 From: castorf <6226927+castorf@users.noreply.github.com> Date: Mon, 21 Nov 2022 14:17:32 -0800 Subject: [PATCH 109/163] update multiple aggregation example. --- docs/cookbook/statistics.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cookbook/statistics.rst b/docs/cookbook/statistics.rst index 51dd3e73..c2fc5643 100644 --- a/docs/cookbook/statistics.rst +++ b/docs/cookbook/statistics.rst @@ -26,9 +26,9 @@ Or, get several at once: .. code-block:: python table.aggregate([ - agate.Min('salary'), - agate.Mean('salary'), - agate.Max('salary') + ('salary_min', agate.Min('salary')), + ('salary_ave', agate.Mean('salary')), + ('salary_max', agate.Max('salary')), ]) Aggregate statistics From 58f107661d6bf90f2605b073c8aa4fe5380972cf Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 16:30:58 -0500 Subject: [PATCH 110/163] ci: Test on Python 3.10 and 3.11 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55e4361e..e5c67291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: os: [macos-latest, windows-latest, ubuntu-latest] - python-version: [2.7, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7] + python-version: [3.7, 3.8, 3.9, '3.10', '3.11', pypy-3.7] exclude: # UnicodeDecodeError on test_to_csv - os: windows-latest From c2282c3692b082b3dd13e16529820bc19725fe06 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 16:36:57 -0500 Subject: [PATCH 111/163] ci: Switch from nose to pytest --- .github/CONTRIBUTING.md | 2 +- .github/workflows/ci.yml | 2 +- docs/contributing.rst | 2 +- docs/install.rst | 2 +- setup.py | 3 ++- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index fcf365db..6cff28f2 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -22,7 +22,7 @@ Contributors should use the following roadmap to guide them through the process 1. Fork the project on [GitHub]. 2. Check out the [issue tracker] and find a task that needs to be done and is of a scope you can realistically expect to complete in a few days. Don’t worry about the priority of the issues at first, but try to choose something you’ll enjoy. You’re much more likely to finish something to the point it can be merged if it’s something you really enjoy hacking on. 3. Comment on the ticket letting everyone know you’re going to be hacking on it so that nobody duplicates your effort. It’s also good practice to provide some general idea of how you plan on resolving the issue so that other developers can make suggestions. -4. Write tests for the feature you’re building. Follow the format of the existing tests in the test directory to see how this works. You can run all the tests with the command `nosetests tests`. +4. Write tests for the feature you’re building. Follow the format of the existing tests in the test directory to see how this works. You can run all the tests with the command `pytest`. 5. Write the code. Try to stay consistent with the style and organization of the existing codebase. A good patch won’t be refused for stylistic reasons, but large parts of it may be rewritten and nobody wants that. 6. As you are coding, periodically merge in work from the master branch and verify you haven’t broken anything by running the test suite. 7. Write documentation. Seriously. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5c67291..fd73089b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: - run: flake8 . - run: isort . --check-only - run: pip install .[test] - - run: nosetests --with-coverage --cover-package=agate + - run: pytest --cov agate # UnicodeDecodeError on print_bars - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' run: python example.py diff --git a/docs/contributing.rst b/docs/contributing.rst index e35cc04f..495083fb 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -30,7 +30,7 @@ Hacker? We'd love to have you hack with us. Please follow this process to make y #. Fork the project on `GitHub `_. #. If you don't have a specific task in mind, check out the `issue tracker `_ and find a task that needs to be done and is of a scope you can realistically expect to complete in a few days. Don't worry about the priority of the issues at first, but try to choose something you'll enjoy. You're much more likely to finish something to the point it can be merged if it's something you really enjoy hacking on. #. If you already have a task you know you want to work on, open a ticket or comment on the existing ticket letting everyone know you're going to be working on it. It's also good practice to provide some general idea of how you plan on resolving the issue so that other developers can make suggestions. -#. Write tests for the feature you're building. Follow the format of the existing tests in the test directory to see how this works. You can run all the tests with the command ``nosetests tests``. +#. Write tests for the feature you're building. Follow the format of the existing tests in the test directory to see how this works. You can run all the tests with the command ``pytest``. #. Write the code. Try to stay consistent with the style and organization of the existing codebase. A good patch won't be refused for stylistic reasons, but large parts of it may be rewritten and nobody wants that. #. As you are coding, periodically merge in work from the master branch and verify you haven't broken anything by running the test suite. #. Write documentation. This means docstrings on all classes and methods, including parameter explanations. It also means, when relevant, cookbook recipes and updates to the agate user tutorial. diff --git a/docs/install.rst b/docs/install.rst index 849ad10e..cb3ecfe8 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -38,7 +38,7 @@ If you are a developer that also wants to hack on agate, install it from git:: To run the agate tests with coverage:: - nosetests --with-coverage tests + pytest --cov agate Supported platforms ------------------- diff --git a/setup.py b/setup.py index fde9583b..6ff305e0 100644 --- a/setup.py +++ b/setup.py @@ -49,9 +49,10 @@ 'coverage>=3.7.1', 'cssselect>=0.9.1', 'lxml>=3.6.0', - 'nose>=1.1.2', # CI is not configured to install PyICU on macOS and Windows. 'PyICU>=2.4.2;sys_platform=="linux"', + 'pytest', + 'pytest-cov', 'pytz>=2015.4', 'mock>=1.3.0;python_version<"3"', 'unittest2>=1.1.0;python_version<"3"', From 47725e4961ab25b83a3ba20aaeabb09e62433240 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 17:23:07 -0500 Subject: [PATCH 112/163] ci: Upgrade workflow and set environment variables for Windows --- .github/workflows/ci.yml | 37 +--- docs/install.rst | 6 +- tests/test_py2.py | 356 --------------------------------------- 3 files changed, 10 insertions(+), 389 deletions(-) delete mode 100644 tests/test_py2.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd73089b..fdd2d117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,15 +8,6 @@ jobs: os: [macos-latest, windows-latest, ubuntu-latest] python-version: [3.7, 3.8, 3.9, '3.10', '3.11', pypy-3.7] exclude: - # UnicodeDecodeError on test_to_csv - - os: windows-latest - python-version: 2.7 - - os: windows-latest - python-version: pypy-2.7 - - os: windows-latest - python-version: 3.6 - - os: windows-latest - python-version: pypy-3.6 - os: windows-latest python-version: 3.7 - os: windows-latest @@ -30,26 +21,16 @@ jobs: sudo locale-gen en_US.UTF-8 sudo locale-gen ko_KR.UTF-8 sudo update-locale - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - # https://github.com/actions/cache/blob/main/examples.md#using-a-script-to-get-cache-location - - id: pip-cache - run: python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" - - uses: actions/cache@v1 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - run: pip install --upgrade check-manifest flake8 isort setuptools - - run: check-manifest - - run: flake8 . - - run: isort . --check-only + cache: pip + cache-dependency-path: setup.py - run: pip install .[test] - - run: pytest --cov agate - # UnicodeDecodeError on print_bars - - if: matrix.os != 'windows-latest' && matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' - run: python example.py + - env: + LANG: en_US.UTF-8 + PYTHONIOENCODING: utf-8 + PYTHONUTF8: 1 + run: pytest --cov agate - run: python charts.py diff --git a/docs/install.rst b/docs/install.rst index cb3ecfe8..fcc14025 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -26,11 +26,7 @@ If you are a developer that also wants to hack on agate, install it from git:: cd agate mkvirtualenv agate - # If running Python 3 (strongly recommended for development) - pip install -r requirements-py3.txt - - # If running Python 2 - pip install -r requirements-py2.txt + pip install -e .[test] python setup.py develop diff --git a/tests/test_py2.py b/tests/test_py2.py deleted file mode 100644 index 817120c6..00000000 --- a/tests/test_py2.py +++ /dev/null @@ -1,356 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import csv -import os - -try: - import unittest2 as unittest -except ImportError: - import unittest - -import six - -from agate import csv_py2 -from agate.exceptions import FieldSizeLimitError - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestUnicodeReader(unittest.TestCase): - def setUp(self): - self.rows = [ - ['number', 'text', 'boolean', 'date', 'datetime', 'timedelta'], - ['1', 'a', 'True', '2015-11-04', '2015-11-04T12:22:00', '0:04:15'], - ['2', u'👍', 'False', '2015-11-05', '2015-11-04T12:45:00', '0:06:18'], - ['', 'b', '', '', '', ''] - ] - - def test_utf8(self): - with open('examples/test.csv') as f: - rows = list(csv_py2.UnicodeReader(f, encoding='utf-8')) - - for a, b in zip(self.rows, rows): - self.assertEqual(a, b) - - def test_latin1(self): - with open('examples/test_latin1.csv') as f: - reader = csv_py2.UnicodeReader(f, encoding='latin1') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'©']) - - def test_utf16_big(self): - with open('examples/test_utf16_big.csv') as f: - reader = csv_py2.UnicodeReader(f, encoding='utf-16') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) - - def test_utf16_little(self): - with open('examples/test_utf16_little.csv') as f: - reader = csv_py2.UnicodeReader(f, encoding='utf-16') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestUnicodeWriter(unittest.TestCase): - def test_utf8(self): - output = six.StringIO() - writer = csv_py2.UnicodeWriter(output, encoding='utf-8') - self.assertEqual(writer._eight_bit, True) - writer.writerow(['a', 'b', 'c']) - writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) - - written = six.StringIO(output.getvalue()) - - reader = csv_py2.UnicodeReader(written, encoding='utf-8') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) - - def test_latin1(self): - output = six.StringIO() - writer = csv_py2.UnicodeWriter(output, encoding='latin1') - self.assertEqual(writer._eight_bit, True) - writer.writerow(['a', 'b', 'c']) - writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'©']) - - written = six.StringIO(output.getvalue()) - - reader = csv_py2.UnicodeReader(written, encoding='latin1') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'©']) - - def test_utf16_big(self): - output = six.StringIO() - writer = csv_py2.UnicodeWriter(output, encoding='utf-16-be') - self.assertEqual(writer._eight_bit, False) - writer.writerow(['a', 'b', 'c']) - writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) - - written = six.StringIO(output.getvalue()) - - reader = csv_py2.UnicodeReader(written, encoding='utf-16-be') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'\u02A4']) - - def test_utf16_little(self): - output = six.StringIO() - writer = csv_py2.UnicodeWriter(output, encoding='utf-16-le') - self.assertEqual(writer._eight_bit, False) - writer.writerow(['a', 'b', 'c']) - writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) - - written = six.StringIO(output.getvalue()) - - reader = csv_py2.UnicodeReader(written, encoding='utf-16-le') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'\u02A4']) - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestUnicodeDictReader(unittest.TestCase): - def setUp(self): - self.rows = [ - ['number', 'text', 'boolean', 'date', 'datetime', 'timedelta'], - ['1', 'a', 'True', '2015-11-04', '2015-11-04T12:22:00', '0:04:15'], - ['2', u'👍', 'False', '2015-11-05', '2015-11-04T12:45:00', '0:06:18'], - ['', 'b', '', '', '', ''] - ] - - self.f = open('examples/test.csv') - - def tearDown(self): - self.f.close() - - def test_reader(self): - reader = csv_py2.UnicodeDictReader(self.f, encoding='utf-8') - - self.assertEqual(next(reader), dict(zip(self.rows[0], self.rows[1]))) - - def test_latin1(self): - with open('examples/test_latin1.csv') as f: - reader = csv_py2.UnicodeDictReader(f, encoding='latin1') - self.assertEqual(next(reader), { - u'a': u'1', - u'b': u'2', - u'c': u'3' - }) - self.assertEqual(next(reader), { - u'a': u'4', - u'b': u'5', - u'c': u'©' - }) - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestUnicodeDictWriter(unittest.TestCase): - def setUp(self): - self.output = six.StringIO() - - def tearDown(self): - self.output.close() - - def test_writer(self): - writer = csv_py2.UnicodeDictWriter(self.output, ['a', 'b', 'c'], lineterminator='\n') - writer.writeheader() - writer.writerow({ - u'a': u'1', - u'b': u'2', - u'c': u'☃' - }) - - result = self.output.getvalue() - - self.assertEqual(result, 'a,b,c\n1,2,☃\n') - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestFieldSizeLimit(unittest.TestCase): - def setUp(self): - self.lim = csv.field_size_limit() - - with open('.test.csv', 'w') as f: - f.write('a' * 10) - - def tearDown(self): - # Resetting limit to avoid failure in other tests. - csv.field_size_limit(self.lim) - os.remove('.test.csv') - - def test_field_size_limit(self): - # Testing field_size_limit for failure. Creating data using str * int. - with open('.test.csv', 'r') as f: - c = csv_py2.UnicodeReader(f, field_size_limit=9) - try: - c.next() - except FieldSizeLimitError: - pass - else: - raise AssertionError('Expected FieldSizeLimitError') - - # Now testing higher field_size_limit. - with open('.test.csv', 'r') as f: - c = csv_py2.UnicodeReader(f, field_size_limit=11) - self.assertEqual(['a' * 10], c.next()) - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestReader(unittest.TestCase): - def setUp(self): - self.rows = [ - ['number', 'text', 'boolean', 'date', 'datetime', 'timedelta'], - ['1', 'a', 'True', '2015-11-04', '2015-11-04T12:22:00', '0:04:15'], - ['2', u'👍', 'False', '2015-11-05', '2015-11-04T12:45:00', '0:06:18'], - ['', 'b', '', '', '', ''] - ] - - def test_utf8(self): - with open('examples/test.csv') as f: - rows = list(csv_py2.Reader(f, encoding='utf-8')) - - for a, b in zip(self.rows, rows): - self.assertEqual(a, b) - - def test_reader_alias(self): - with open('examples/test.csv') as f: - rows = list(csv_py2.Reader(f, encoding='utf-8')) - - for a, b in zip(self.rows, rows): - self.assertEqual(a, b) - - def test_line_numbers(self): - with open('examples/test.csv') as f: - rows = list(csv_py2.Reader(f, encoding='utf-8', line_numbers=True)) - - sample_rows = [ - ['line_numbers', 'number', 'text', 'boolean', 'date', 'datetime', 'timedelta'], - ['1', '1', 'a', 'True', '2015-11-04', '2015-11-04T12:22:00', '0:04:15'], - ['2', '2', u'👍', 'False', '2015-11-05', '2015-11-04T12:45:00', '0:06:18'], - ['3', '', 'b', '', '', '', ''] - ] - - for a, b in zip(sample_rows, rows): - self.assertEqual(a, b) - - def test_properties(self): - with open('examples/test.csv') as f: - reader = csv_py2.Reader(f, encoding='utf-8') - - self.assertEqual(reader.dialect.delimiter, ',') - self.assertEqual(reader.line_num, 0) - - next(reader) - - self.assertEqual(reader.line_num, 1) - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestWriter(unittest.TestCase): - def test_utf8(self): - output = six.StringIO() - writer = csv_py2.Writer(output, encoding='utf-8') - self.assertEqual(writer._eight_bit, True) - writer.writerow(['a', 'b', 'c']) - writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) - - written = six.StringIO(output.getvalue()) - - reader = csv_py2.Reader(written, encoding='utf-8') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) - - def test_writer_alias(self): - output = six.StringIO() - writer = csv_py2.writer(output, encoding='utf-8') - self.assertEqual(writer._eight_bit, True) - writer.writerow(['a', 'b', 'c']) - writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) - - written = six.StringIO(output.getvalue()) - - reader = csv_py2.reader(written, encoding='utf-8') - self.assertEqual(next(reader), ['a', 'b', 'c']) - self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestDictReader(unittest.TestCase): - def setUp(self): - self.rows = [ - ['number', 'text', 'boolean', 'date', 'datetime', 'timedelta'], - ['1', 'a', 'True', '2015-11-04', '2015-11-04T12:22:00', '0:04:15'], - ['2', u'👍', 'False', '2015-11-05', '2015-11-04T12:45:00', '0:06:18'], - ['', 'b', '', '', '', ''] - ] - - self.f = open('examples/test.csv') - - def tearDown(self): - self.f.close() - - def test_reader(self): - reader = csv_py2.DictReader(self.f, encoding='utf-8') - - self.assertEqual(next(reader), dict(zip(self.rows[0], self.rows[1]))) - - def test_reader_alias(self): - reader = csv_py2.DictReader(self.f, encoding='utf-8') - - self.assertEqual(next(reader), dict(zip(self.rows[0], self.rows[1]))) - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestDictWriter(unittest.TestCase): - def setUp(self): - self.output = six.StringIO() - - def tearDown(self): - self.output.close() - - def test_writer(self): - writer = csv_py2.DictWriter(self.output, ['a', 'b', 'c']) - writer.writeheader() - writer.writerow({ - u'a': u'1', - u'b': u'2', - u'c': u'☃' - }) - - result = self.output.getvalue() - - self.assertEqual(result, 'a,b,c\n1,2,☃\n') - - def test_writer_alias(self): - writer = csv_py2.DictWriter(self.output, ['a', 'b', 'c']) - writer.writeheader() - writer.writerow({ - u'a': u'1', - u'b': u'2', - u'c': u'☃' - }) - - result = self.output.getvalue() - - self.assertEqual(result, 'a,b,c\n1,2,☃\n') - - -@unittest.skipIf(six.PY3, "Not supported in Python 3.") -class TestSniffer(unittest.TestCase): - def test_sniffer(self): - with open('examples/test.csv') as f: - contents = f.read() - self.assertEqual(csv_py2.Sniffer().sniff(contents).__dict__, csv.Sniffer().sniff(contents).__dict__) From 5c8440220c972891ed89273f4b2abbf49a4082c4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 17:23:44 -0500 Subject: [PATCH 113/163] ci: Move linting to separate workflow --- .github/workflows/lint.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..28c49bb4 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,19 @@ +name: Lint +on: [push, pull_request] +env: + BASEDIR: https://raw.githubusercontent.com/open-contracting/standard-maintenance-scripts/main +jobs: + build: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: setup.py + - run: pip install --upgrade check-manifest flake8 isort setuptools + - run: check-manifest + - run: flake8 . + - run: isort . --check-only From c83d726d535dcec08b0c5b1301b8671068c25b12 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:03:36 -0500 Subject: [PATCH 114/163] chore: Remove Python 2 code --- agate/__init__.py | 7 +---- agate/aggregations/base.py | 4 +-- agate/columns.py | 5 ---- agate/computations/base.py | 5 +--- agate/computations/rank.py | 11 ++------ agate/csv_py2.py | 21 +++++++------- agate/csv_py3.py | 12 ++++---- agate/data_types/base.py | 7 ++--- agate/data_types/boolean.py | 9 ++---- agate/data_types/date.py | 3 +- agate/data_types/date_time.py | 3 +- agate/data_types/number.py | 11 ++------ agate/data_types/text.py | 5 ++-- agate/data_types/time_delta.py | 3 +- agate/fixed.py | 4 +-- agate/mapped_sequence.py | 12 +------- agate/table/__init__.py | 13 ++++----- agate/table/bins.py | 5 +--- agate/table/denormalize.py | 10 ++----- agate/table/from_csv.py | 15 +++------- agate/table/pivot.py | 5 ++-- agate/table/print_bars.py | 12 ++------ agate/table/print_html.py | 3 +- agate/table/print_table.py | 3 +- agate/table/to_json.py | 13 ++------- agate/tableset/__init__.py | 6 ++-- agate/tableset/from_json.py | 6 ++-- agate/tableset/to_json.py | 8 ++---- agate/testcase.py | 5 +--- agate/type_tester.py | 2 +- agate/utils.py | 25 +++++----------- benchmarks/test_joins.py | 13 ++------- docs/cookbook/compute.rst | 3 +- docs/install.rst | 6 ---- setup.py | 3 -- tests/test_agate.py | 21 ++++---------- tests/test_aggregations.py | 6 +--- tests/test_columns.py | 12 ++------ tests/test_computations.py | 6 +--- tests/test_data_types.py | 12 +------- tests/test_fixed.py | 5 +--- tests/test_from_json.py | 9 ++---- tests/test_mapped_sequence.py | 17 ++--------- tests/test_py3.py | 36 +++++++++--------------- tests/test_table/__init__.py | 29 ++++--------------- tests/test_table/test_bins.py | 7 ++--- tests/test_table/test_compute.py | 5 ++-- tests/test_table/test_from_csv.py | 7 +---- tests/test_table/test_group_by.py | 5 +--- tests/test_table/test_homogenize.py | 2 -- tests/test_table/test_pivot.py | 6 +--- tests/test_table/test_print_bars.py | 11 ++++---- tests/test_table/test_print_html.py | 17 ++++++----- tests/test_table/test_print_structure.py | 4 +-- tests/test_table/test_print_table.py | 17 +++++------ tests/test_table/test_to_csv.py | 7 ++--- tests/test_table/test_to_json.py | 19 ++++++------- tests/test_tableset/__init__.py | 9 ++---- tests/test_tableset/test_aggregate.py | 6 +--- tests/test_type_tester.py | 5 +--- tests/test_utils.py | 11 ++------ 61 files changed, 159 insertions(+), 410 deletions(-) diff --git a/agate/__init__.py b/agate/__init__.py index 910eb7dc..ae6b4b2b 100644 --- a/agate/__init__.py +++ b/agate/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import six +import agate.csv_py3 as csv from agate.aggregations import * from agate.columns import Column from agate.computations import * @@ -17,8 +17,3 @@ from agate.type_tester import TypeTester from agate.utils import * from agate.warns import DuplicateColumnWarning, NullCalculationWarning, warn_duplicate_column, warn_null_calculation - -if six.PY2: # pragma: no cover - import agate.csv_py2 as csv -else: - import agate.csv_py3 as csv diff --git a/agate/aggregations/base.py b/agate/aggregations/base.py index 4173ac61..7a440897 100644 --- a/agate/aggregations/base.py +++ b/agate/aggregations/base.py @@ -1,12 +1,10 @@ #!/usr/bin/env python -import six from agate.exceptions import UnsupportedAggregationError -@six.python_2_unicode_compatible -class Aggregation(object): # pragma: no cover +class Aggregation: # pragma: no cover """ Aggregations create a new value by summarizing a :class:`.Column`. diff --git a/agate/columns.py b/agate/columns.py index 7e556c3e..3b99307a 100644 --- a/agate/columns.py +++ b/agate/columns.py @@ -6,15 +6,10 @@ parent :class:`.Table`, columns depend on knowledge of both their position in the parent (column name, data type) as well as the rows that contain their data. """ -import six from agate.mapped_sequence import MappedSequence from agate.utils import NullOrder, memoize -if six.PY3: # pragma: no cover - # pylint: disable=W0622 - xrange = range - def null_handler(k): """ diff --git a/agate/computations/base.py b/agate/computations/base.py index 6ea05182..7de4463c 100644 --- a/agate/computations/base.py +++ b/agate/computations/base.py @@ -1,10 +1,7 @@ #!/usr/bin/env python -import six - -@six.python_2_unicode_compatible -class Computation(object): # pragma: no cover +class Computation: # pragma: no cover """ Computations produce a new column by performing a calculation on each row. diff --git a/agate/computations/rank.py b/agate/computations/rank.py index 46cec5c9..a6fe88b2 100644 --- a/agate/computations/rank.py +++ b/agate/computations/rank.py @@ -1,11 +1,7 @@ #!/usr/bin/env python from decimal import Decimal - -import six - -if six.PY3: - from functools import cmp_to_key +from functools import cmp_to_key from agate.computations.base import Computation from agate.data_types import Number @@ -44,10 +40,7 @@ def run(self, table): column = table.columns[self._column_name] if self._comparer: - if six.PY3: - data_sorted = sorted(column.values(), key=cmp_to_key(self._comparer)) - else: # pragma: no cover - data_sorted = sorted(column.values(), cmp=self._comparer) + data_sorted = sorted(column.values(), key=cmp_to_key(self._comparer)) else: data_sorted = column.values_sorted() diff --git a/agate/csv_py2.py b/agate/csv_py2.py index ea30f77f..568d2603 100644 --- a/agate/csv_py2.py +++ b/agate/csv_py2.py @@ -7,8 +7,7 @@ import codecs import csv import warnings - -import six +from io import StringIO from agate.exceptions import FieldSizeLimitError @@ -20,7 +19,7 @@ POSSIBLE_DELIMITERS = [',', '\t', ';', ' ', ':', '|'] -class UTF8Recoder(six.Iterator): +class UTF8Recoder: """ Iterator that reads an encoded stream and reencodes the input to UTF-8. """ @@ -34,7 +33,7 @@ def __next__(self): return next(self.reader).encode('utf-8') -class UnicodeReader(object): +class UnicodeReader: """ A CSV reader which will read rows from a file in a given encoding. """ @@ -65,7 +64,7 @@ def next(self): else: row.insert(0, str(self.line_num - 1 if self.header else self.line_num)) - return [six.text_type(s, 'utf-8') for s in row] + return [str(s, 'utf-8') for s in row] def __iter__(self): return self @@ -79,7 +78,7 @@ def line_num(self): return self.reader.line_num -class UnicodeWriter(object): +class UnicodeWriter: """ A CSV writer which will write rows to a file in the specified encoding. @@ -94,16 +93,16 @@ def __init__(self, f, encoding='utf-8', **kwargs): self.writer = csv.writer(f, **kwargs) else: # Redirect output to a queue for reencoding - self.queue = six.StringIO() + self.queue = StringIO() self.writer = csv.writer(self.queue, **kwargs) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() def writerow(self, row): if self._eight_bit: - self.writer.writerow([six.text_type(s if s is not None else '').encode(self.encoding) for s in row]) + self.writer.writerow([str(s if s is not None else '').encode(self.encoding) for s in row]) else: - self.writer.writerow([six.text_type(s if s is not None else '').encode('utf-8') for s in row]) + self.writer.writerow([str(s if s is not None else '').encode('utf-8') for s in row]) # Fetch UTF-8 output from the queue... data = self.queue.getvalue() data = data.decode('utf-8') @@ -186,7 +185,7 @@ def writerow(self, row): self._append_line_number(row) # Convert embedded Mac line endings to unix style line endings so they get quoted - row = [i.replace('\r', '\n') if isinstance(i, six.string_types) else i for i in row] + row = [i.replace('\r', '\n') if isinstance(i, str) else i for i in row] UnicodeWriter.writerow(self, row) @@ -240,7 +239,7 @@ def writerows(self, rows): self.writerow(row) -class Sniffer(object): +class Sniffer: """ A functional wrapper of ``csv.Sniffer()``. """ diff --git a/agate/csv_py3.py b/agate/csv_py3.py index 94354882..9f2761f7 100644 --- a/agate/csv_py3.py +++ b/agate/csv_py3.py @@ -7,14 +7,12 @@ import csv import warnings -import six - from agate.exceptions import FieldSizeLimitError POSSIBLE_DELIMITERS = [',', '\t', ';', ' ', ':', '|'] -class Reader(six.Iterator): +class Reader: """ A wrapper around Python 3's builtin :func:`csv.reader`. """ @@ -60,7 +58,7 @@ def line_num(self): return self.reader.line_num -class Writer(object): +class Writer: """ A wrapper around Python 3's builtin :func:`csv.writer`. """ @@ -87,7 +85,7 @@ def writerow(self, row): self._append_line_number(row) # Convert embedded Mac line endings to unix style line endings so they get quoted - row = [i.replace('\r', '\n') if isinstance(i, six.string_types) else i for i in row] + row = [i.replace('\r', '\n') if isinstance(i, str) else i for i in row] self.writer.writerow(row) @@ -129,7 +127,7 @@ def _append_line_number(self, row): def writerow(self, row): # Convert embedded Mac line endings to unix style line endings so they get quoted - row = dict([(k, v.replace('\r', '\n')) if isinstance(v, six.string_types) else (k, v) for k, v in row.items()]) + row = dict([(k, v.replace('\r', '\n')) if isinstance(v, str) else (k, v) for k, v in row.items()]) if self.line_numbers: self._append_line_number(row) @@ -141,7 +139,7 @@ def writerows(self, rows): self.writerow(row) -class Sniffer(object): +class Sniffer: """ A functional wrapper of ``csv.Sniffer()``. """ diff --git a/agate/data_types/base.py b/agate/data_types/base.py index de7eeb5a..fe583028 100644 --- a/agate/data_types/base.py +++ b/agate/data_types/base.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import six from agate.exceptions import CastError @@ -8,7 +7,7 @@ DEFAULT_NULL_VALUES = ('', 'na', 'n/a', 'none', 'null', '.') -class DataType(object): # pragma: no cover +class DataType: # pragma: no cover """ Specifies how values should be parsed when creating a :class:`.Table`. @@ -45,7 +44,7 @@ def csvify(self, d): if d is None: return None - return six.text_type(d) + return str(d) def jsonify(self, d): """ @@ -54,4 +53,4 @@ def jsonify(self, d): if d is None: return None - return six.text_type(d) + return str(d) diff --git a/agate/data_types/boolean.py b/agate/data_types/boolean.py index 05791156..4b9b2341 100644 --- a/agate/data_types/boolean.py +++ b/agate/data_types/boolean.py @@ -1,11 +1,6 @@ #!/usr/bin/env python -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal - -import six +from decimal import Decimal from agate.data_types.base import DEFAULT_NULL_VALUES, DataType from agate.exceptions import CastError @@ -52,7 +47,7 @@ def cast(self, d): return True elif d == 0: return False - elif isinstance(d, six.string_types): + elif isinstance(d, str): d = d.replace(',', '').strip() d_lower = d.lower() diff --git a/agate/data_types/date.py b/agate/data_types/date.py index 20bdb6e4..99b526fd 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -4,7 +4,6 @@ from datetime import date, datetime, time import parsedatetime -import six from agate.data_types.base import DataType from agate.exceptions import CastError @@ -64,7 +63,7 @@ def cast(self, d): """ if type(d) is date or d is None: return d - elif isinstance(d, six.string_types): + elif isinstance(d, str): d = d.strip() if d.lower() in self.null_values: diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index 8cb9660a..669cab38 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -5,7 +5,6 @@ import isodate import parsedatetime -import six from agate.data_types.base import DataType from agate.exceptions import CastError @@ -73,7 +72,7 @@ def cast(self, d): return d elif isinstance(d, datetime.date): return datetime.datetime.combine(d, datetime.time(0, 0, 0)) - elif isinstance(d, six.string_types): + elif isinstance(d, str): d = d.strip() if d.lower() in self.null_values: diff --git a/agate/data_types/number.py b/agate/data_types/number.py index c875cb3a..e41e8fa4 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -1,14 +1,9 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -try: - from cdecimal import Decimal, InvalidOperation -except ImportError: # pragma: no cover - from decimal import Decimal, InvalidOperation - import warnings +from decimal import Decimal, InvalidOperation -import six from babel.core import Locale from agate.data_types.base import DataType @@ -68,15 +63,13 @@ def cast(self, d): if t is int: return Decimal(d) - elif six.PY2 and t is long: # noqa: F821 - return Decimal(d) elif t is float: return Decimal(repr(d)) elif d is False: return Decimal(0) elif d is True: return Decimal(1) - elif not isinstance(d, six.string_types): + elif not isinstance(d, str): raise CastError('Can not parse value "%s" as Decimal.' % d) d = d.strip() diff --git a/agate/data_types/text.py b/agate/data_types/text.py index 6bd210ea..ed31bef5 100644 --- a/agate/data_types/text.py +++ b/agate/data_types/text.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import six from agate.data_types.base import DataType @@ -29,8 +28,8 @@ def cast(self, d): """ if d is None: return d - elif isinstance(d, six.string_types): + elif isinstance(d, str): if self.cast_nulls and d.strip().lower() in self.null_values: return None - return six.text_type(d) + return str(d) diff --git a/agate/data_types/time_delta.py b/agate/data_types/time_delta.py index a577a81b..e6e49718 100644 --- a/agate/data_types/time_delta.py +++ b/agate/data_types/time_delta.py @@ -3,7 +3,6 @@ import datetime import pytimeparse -import six from agate.data_types.base import DataType from agate.exceptions import CastError @@ -24,7 +23,7 @@ def cast(self, d): """ if isinstance(d, datetime.timedelta) or d is None: return d - elif isinstance(d, six.string_types): + elif isinstance(d, str): d = d.strip() if d.lower() in self.null_values: diff --git a/agate/fixed.py b/agate/fixed.py index 66f4fad9..6b9e12ae 100644 --- a/agate/fixed.py +++ b/agate/fixed.py @@ -7,12 +7,10 @@ from collections import OrderedDict, namedtuple -import six - Field = namedtuple('Field', ['name', 'start', 'length']) -class Reader(six.Iterator): +class Reader: """ Reads a fixed-width file using a column schema in CSV format. diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index 403528d1..3b4c3cd0 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -7,14 +7,7 @@ """ from collections import OrderedDict - -try: - from collections.abc import Sequence -except ImportError: - from collections import Sequence - -import six -from six.moves import range # pylint: disable=W0622 +from collections.abc import Sequence from agate.utils import memoize @@ -78,9 +71,6 @@ def __str__(self): """ Print an ascii sample of the contents of this sequence. """ - if six.PY2: # pragma: no cover - return str(self.__unicode__().encode('utf8')) - return str(self.__unicode__()) def __repr__(self): diff --git a/agate/table/__init__.py b/agate/table/__init__.py index 3affe053..918f6478 100644 --- a/agate/table/__init__.py +++ b/agate/table/__init__.py @@ -21,11 +21,9 @@ import sys import warnings +from io import StringIO from itertools import chain -import six -from six.moves import range # pylint: disable=W0622 - from agate import utils from agate.columns import Column from agate.data_types import DataType @@ -35,8 +33,7 @@ from agate.type_tester import TypeTester -@six.python_2_unicode_compatible -class Table(object): +class Table: """ A dataset consisting of rows and columns. Columns refer to "vertical" slices of data that must all be of the same type. Rows refer to "horizontal" slices @@ -76,7 +73,7 @@ class Table(object): assumed to be :class:`.Row` instances, rather than raw data. """ def __init__(self, rows, column_names=None, column_types=None, row_names=None, _is_fork=False): - if isinstance(rows, six.string_types): + if isinstance(rows, str): raise ValueError('When created directly, the first argument to Table must be a sequence of rows. ' 'Did you want agate.Table.from_csv?') @@ -142,7 +139,7 @@ def __init__(self, rows, column_names=None, column_types=None, row_names=None, _ if row_names: computed_row_names = [] - if isinstance(row_names, six.string_types): + if isinstance(row_names, str): for row in new_rows: name = row[row_names] computed_row_names.append(name) @@ -182,7 +179,7 @@ def __str__(self): """ Print the table's structure using :meth:`.Table.print_structure`. """ - structure = six.StringIO() + structure = StringIO() self.print_structure(output=structure) diff --git a/agate/table/bins.py b/agate/table/bins.py index 0acc9a83..ee14d81c 100644 --- a/agate/table/bins.py +++ b/agate/table/bins.py @@ -1,10 +1,7 @@ #!/usr/bin/env python # pylint: disable=W0212 -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal +from decimal import Decimal from babel.numbers import format_decimal diff --git a/agate/table/denormalize.py b/agate/table/denormalize.py index 03e07953..1a571261 100644 --- a/agate/table/denormalize.py +++ b/agate/table/denormalize.py @@ -2,13 +2,7 @@ # pylint: disable=W0212 from collections import OrderedDict - -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal - -import six +from decimal import Decimal from agate import utils from agate.data_types import Number @@ -91,7 +85,7 @@ def denormalize(self, key=None, property_column='property', value_column='value' if row_key not in row_data: row_data[row_key] = OrderedDict() - f = six.text_type(row[property_column]) + f = str(row[property_column]) v = row[value_column] if f not in field_names: diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index 81daa72b..e6ae7288 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -2,8 +2,7 @@ import io import itertools - -import six +from io import StringIO @classmethod @@ -52,10 +51,7 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk if hasattr(path, 'read'): f = path else: - if six.PY2: - f = open(path, 'Urb') - else: - f = io.open(path, encoding=encoding) + f = io.open(path, encoding=encoding) close = True @@ -66,17 +62,14 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk else: raise ValueError('skip_lines argument must be an int') - contents = six.StringIO(f.read()) + contents = StringIO(f.read()) if sniff_limit is None: kwargs['dialect'] = csv.Sniffer().sniff(contents.getvalue()) elif sniff_limit > 0: kwargs['dialect'] = csv.Sniffer().sniff(contents.getvalue()[:sniff_limit]) - if six.PY2: - kwargs['encoding'] = encoding - - reader = csv.reader(contents, header=header, **kwargs) + reader = csv.reader(contents, header=header, encoding=encoding, **kwargs) if header: if column_names is None: diff --git a/agate/table/pivot.py b/agate/table/pivot.py index 9b259cb1..ae6cfe42 100644 --- a/agate/table/pivot.py +++ b/agate/table/pivot.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # pylint: disable=W0212 -import six from agate import utils from agate.aggregations import Count @@ -95,8 +94,8 @@ def pivot(self, key=None, pivot=None, aggregation=None, computation=None, defaul for k in key: groups = groups.group_by(k, key_name=key_name) - aggregation_name = six.text_type(aggregation) - computation_name = six.text_type(computation) if computation else None + aggregation_name = str(aggregation) + computation_name = str(computation) if computation else None def apply_computation(table): computed = table.compute([ diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index a7aadaf3..739fc391 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -2,16 +2,10 @@ # -*- coding: utf8 -*- # pylint: disable=W0212 -from collections import OrderedDict - -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal - import sys +from collections import OrderedDict +from decimal import Decimal -import six from babel.numbers import format_decimal from agate import config, utils @@ -76,7 +70,7 @@ def print_bars(self, label_column_name='group', value_column_name='Count', domai formatted_labels = [] for label in label_column: - formatted_labels.append(six.text_type(label)) + formatted_labels.append(str(label)) formatted_values = [] for value in value_column: diff --git a/agate/table/print_html.py b/agate/table/print_html.py index 1639925a..dd5cca86 100644 --- a/agate/table/print_html.py +++ b/agate/table/print_html.py @@ -4,7 +4,6 @@ import math import sys -import six from babel.numbers import format_decimal from agate import config, utils @@ -94,7 +93,7 @@ def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_w locale=locale ) else: - v = six.text_type(v) + v = str(v) if max_column_width is not None and len(v) > max_column_width: v = '%s...' % v[:max_column_width - 3] diff --git a/agate/table/print_table.py b/agate/table/print_table.py index 101ec5b3..eece96ab 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -4,7 +4,6 @@ import math import sys -import six from babel.numbers import format_decimal from agate import config, utils @@ -103,7 +102,7 @@ def print_table(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_ locale=locale ) else: - v = six.text_type(v) + v = str(v) if max_column_width is not None and len(v) > max_column_width: v = '%s...' % v[:max_column_width - 3] diff --git a/agate/table/to_json.py b/agate/table/to_json.py index 01333bd6..7686a419 100644 --- a/agate/table/to_json.py +++ b/agate/table/to_json.py @@ -1,13 +1,10 @@ #!/usr/bin/env python # pylint: disable=W0212 -import codecs import json import os from collections import OrderedDict -import six - def to_json(self, path, key=None, newline=False, indent=None, **kwargs): """ @@ -41,9 +38,6 @@ def to_json(self, path, key=None, newline=False, indent=None, **kwargs): 'indent': indent } - if six.PY2: - json_kwargs['encoding'] = 'utf-8' - # Pass remaining kwargs through to JSON encoder json_kwargs.update(kwargs) @@ -61,9 +55,6 @@ def to_json(self, path, key=None, newline=False, indent=None, **kwargs): os.makedirs(os.path.dirname(path)) f = open(path, 'w') - if six.PY2: - f = codecs.getwriter('utf-8')(f) - def dump_json(data): json.dump(data, f, **json_kwargs) @@ -78,10 +69,10 @@ def dump_json(data): if key_is_row_function: k = key(row) else: - k = str(row[key]) if six.PY3 else unicode(row[key]) # noqa: F821 + k = str(row[key]) if k in output: - raise ValueError('Value %s is not unique in the key column.' % six.text_type(k)) + raise ValueError('Value %s is not unique in the key column.' % str(k)) values = tuple(json_funcs[i](d) for i, d in enumerate(row)) output[k] = OrderedDict(zip(row.keys(), values)) diff --git a/agate/tableset/__init__.py b/agate/tableset/__init__.py index 5dbe58bf..972578af 100644 --- a/agate/tableset/__init__.py +++ b/agate/tableset/__init__.py @@ -24,8 +24,8 @@ dimensions. """ -import six -from six.moves import zip_longest +from io import StringIO +from itertools import zip_longest from agate.data_types import Text from agate.mapped_sequence import MappedSequence @@ -86,7 +86,7 @@ def __str__(self): """ Print the tableset's structure via :meth:`TableSet.print_structure`. """ - structure = six.StringIO() + structure = StringIO() self.print_structure(output=structure) diff --git a/agate/tableset/from_json.py b/agate/tableset/from_json.py index cd069120..e83205ef 100644 --- a/agate/tableset/from_json.py +++ b/agate/tableset/from_json.py @@ -6,8 +6,6 @@ from decimal import Decimal from glob import glob -import six - from agate.table import Table @@ -31,12 +29,12 @@ def from_json(cls, path, column_names=None, column_types=None, keys=None, **kwar """ from agate.tableset import TableSet - if isinstance(path, six.string_types) and not os.path.isdir(path) and not os.path.isfile(path): + if isinstance(path, str) and not os.path.isdir(path) and not os.path.isfile(path): raise IOError('Specified path doesn\'t exist.') tables = OrderedDict() - if isinstance(path, six.string_types) and os.path.isdir(path): + if isinstance(path, str) and os.path.isdir(path): filepaths = glob(os.path.join(path, '*.json')) if keys is not None and len(keys) != len(filepaths): diff --git a/agate/tableset/to_json.py b/agate/tableset/to_json.py index 963bab8a..dea7916c 100644 --- a/agate/tableset/to_json.py +++ b/agate/tableset/to_json.py @@ -3,8 +3,7 @@ import json import os from collections import OrderedDict - -import six +from io import StringIO def to_json(self, path, nested=False, indent=None, **kwargs): @@ -37,7 +36,7 @@ def to_json(self, path, nested=False, indent=None, **kwargs): tableset_dict = OrderedDict() for name, table in self.items(): - output = six.StringIO() + output = StringIO() table.to_json(output, **kwargs) tableset_dict[name] = json.loads(output.getvalue(), object_pairs_hook=OrderedDict) @@ -54,9 +53,6 @@ def to_json(self, path, nested=False, indent=None, **kwargs): json_kwargs = {'ensure_ascii': False, 'indent': indent} - if six.PY2: - json_kwargs['encoding'] = 'utf-8' - json_kwargs.update(kwargs) json.dump(tableset_dict, f, **json_kwargs) diff --git a/agate/testcase.py b/agate/testcase.py index 89438200..71db94ee 100644 --- a/agate/testcase.py +++ b/agate/testcase.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import agate diff --git a/agate/type_tester.py b/agate/type_tester.py index 8f92bbfc..3c82210a 100644 --- a/agate/type_tester.py +++ b/agate/type_tester.py @@ -12,7 +12,7 @@ from agate.data_types.time_delta import TimeDelta -class TypeTester(object): +class TypeTester: """ Control how data types are inferred for columns in a given set of data. diff --git a/agate/utils.py b/agate/utils.py index 3272ab73..8171c529 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -6,28 +6,17 @@ agate. """ -from collections import OrderedDict - -try: - from collections.abc import Sequence -except ImportError: - from collections import Sequence - import math import string +from collections import OrderedDict +from collections.abc import Sequence +from decimal import ROUND_CEILING, ROUND_FLOOR, Decimal, getcontext from functools import wraps from slugify import slugify as pslugify from agate.warns import warn_duplicate_column, warn_unnamed_column -try: - from cdecimal import ROUND_CEILING, ROUND_FLOOR, Decimal, getcontext -except ImportError: # pragma: no cover - from decimal import Decimal, ROUND_FLOOR, ROUND_CEILING, getcontext - -import six - #: Sentinal for use when `None` is an valid argument value default = object() @@ -50,7 +39,7 @@ def wrapper(self): return wrapper -class NullOrder(object): +class NullOrder: """ Dummy object used for sorting in place of None. @@ -249,7 +238,7 @@ def parse_object(obj, path=''): d = OrderedDict() for key, value in iterator: - key = six.text_type(key) + key = str(key) d.update(parse_object(value, path + key + '/')) return d @@ -260,7 +249,7 @@ def issequence(obj): Returns :code:`True` if the given object is an instance of :class:`.Sequence` that is not also a string. """ - return isinstance(obj, Sequence) and not isinstance(obj, six.string_types) + return isinstance(obj, Sequence) and not isinstance(obj, str) def deduplicate(values, column_names=False, separator='_'): @@ -283,7 +272,7 @@ def deduplicate(values, column_names=False, separator='_'): if not value: new_value = letter_name(i) warn_unnamed_column(i, new_value) - elif isinstance(value, six.string_types): + elif isinstance(value, str): new_value = value else: raise ValueError('Column names must be strings or None.') diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index b26d0da7..211df19d 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -1,24 +1,17 @@ #!/usr/bin/env python # -*- coding: utf8 -*- +import unittest from random import shuffle from timeit import Timer -try: - import unittest2 as unittest -except ImportError: - import unittest - -import six -from six.moves import range - import agate class TestTableJoin(unittest.TestCase): def test_join(self): - left_rows = [(six.text_type(i), i) for i in range(100000)] - right_rows = [(six.text_type(i), i) for i in range(100000)] + left_rows = [(str(i), i) for i in range(100000)] + right_rows = [(str(i), i) for i in range(100000)] shuffle(left_rows) shuffle(right_rows) diff --git a/docs/cookbook/compute.rst b/docs/cookbook/compute.rst index 738aecda..725eb76e 100644 --- a/docs/cookbook/compute.rst +++ b/docs/cookbook/compute.rst @@ -145,8 +145,7 @@ Implementing Levenshtein requires writing a custom :class:`.Computation`. To sav import agate from Levenshtein import distance - import six - + class LevenshteinDistance(agate.Computation): """ Computes Levenshtein edit distance between the column and a given string. diff --git a/docs/install.rst b/docs/install.rst index fcc14025..d86badc6 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -11,12 +11,6 @@ To use agate install it with pip:: For non-English locale support, `install PyICU `__. -.. note:: - - Need more speed? Upgrade to Python 3. It's 3-5x faster than Python 2. - - If you must use Python 2 you can you can :code:`pip install cdecimal` for a performance boost. - Developers ---------- diff --git a/setup.py b/setup.py index 6ff305e0..bfe38ca1 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ 'parsedatetime>=2.1,!=2.5,!=2.6', 'python-slugify>=1.2.1', 'pytimeparse>=1.1.5', - 'six>=1.9.0', ], extras_require={ 'test': [ @@ -54,8 +53,6 @@ 'pytest', 'pytest-cov', 'pytz>=2015.4', - 'mock>=1.3.0;python_version<"3"', - 'unittest2>=1.1.0;python_version<"3"', ], 'docs': [ 'Sphinx>=1.2.2', diff --git a/tests/test_agate.py b/tests/test_agate.py index cd6e4fcc..41fbe7d7 100644 --- a/tests/test_agate.py +++ b/tests/test_agate.py @@ -1,24 +1,13 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest - -import six +import unittest import agate class TestCSV(unittest.TestCase): def test_agate(self): - if six.PY2: - self.assertIs(agate.csv.reader, agate.csv_py2.reader) - self.assertIs(agate.csv.writer, agate.csv_py2.writer) - self.assertIs(agate.csv.DictReader, agate.csv_py2.DictReader) - self.assertIs(agate.csv.DictWriter, agate.csv_py2.DictWriter) - else: - self.assertIs(agate.csv.reader, agate.csv_py3.reader) - self.assertIs(agate.csv.writer, agate.csv_py3.writer) - self.assertIs(agate.csv.DictReader, agate.csv_py3.DictReader) - self.assertIs(agate.csv.DictWriter, agate.csv_py3.DictWriter) + self.assertIs(agate.csv.reader, agate.csv_py3.reader) + self.assertIs(agate.csv.writer, agate.csv_py3.writer) + self.assertIs(agate.csv.DictReader, agate.csv_py3.DictReader) + self.assertIs(agate.csv.DictWriter, agate.csv_py3.DictWriter) diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 32479cb6..5ce4c659 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -3,14 +3,10 @@ import datetime import sys +import unittest import warnings from decimal import Decimal -try: - import unittest2 as unittest -except ImportError: - import unittest - from agate import Table from agate.aggregations import (IQR, MAD, All, Any, Count, Deciles, First, HasNulls, Max, MaxLength, MaxPrecision, Mean, Median, Min, Mode, Percentiles, PopulationStDev, PopulationVariance, Quartiles, diff --git a/tests/test_columns.py b/tests/test_columns.py index 107219e7..2b04e311 100644 --- a/tests/test_columns.py +++ b/tests/test_columns.py @@ -2,16 +2,8 @@ # -*- coding: utf8 -*- import pickle - -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal - -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest +from decimal import Decimal from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_computations.py b/tests/test_computations.py index df1857e0..c7854ff6 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -1,14 +1,10 @@ #!/usr/bin/env Python import datetime +import unittest import warnings from decimal import Decimal -try: - import unittest2 as unittest -except ImportError: - import unittest - from agate import Table from agate.computations import Change, Formula, Percent, PercentChange, PercentileRank, Rank, Slug from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta diff --git a/tests/test_data_types.py b/tests/test_data_types.py index e284c741..0f65571a 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -3,16 +3,11 @@ import datetime import pickle +import unittest from decimal import Decimal -try: - import unittest2 as unittest -except ImportError: - import unittest - import parsedatetime import pytz -import six from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.exceptions import CastError @@ -154,11 +149,6 @@ def test_cast(self): (Decimal('2'), Decimal('1'), None, Decimal('2.7'), None, Decimal('2.7'), Decimal('200000000')) ) - @unittest.skipIf(six.PY3, 'Not supported in Python 3.') - def test_cast_long(self): - self.assertEqual(self.type.test(long('141414')), True) # noqa: F821 - self.assertEqual(self.type.cast(long('141414')), Decimal('141414')) # noqa: F821 - def test_boolean_cast(self): values = (True, False) casted = tuple(self.type.cast(v) for v in values) diff --git a/tests/test_fixed.py b/tests/test_fixed.py index 5ce42b56..18973d6e 100644 --- a/tests/test_fixed.py +++ b/tests/test_fixed.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate import csv, fixed diff --git a/tests/test_from_json.py b/tests/test_from_json.py index 471dccc8..ea191db6 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import six from agate import Table from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta @@ -36,12 +35,8 @@ def test_from_json(self): def test_from_json_file_like_object(self): table1 = Table(self.rows, self.column_names, self.column_types) - if six.PY2: - with open('examples/test.json') as f: - table2 = Table.from_json(f) - else: - with open('examples/test.json', encoding='utf-8') as f: - table2 = Table.from_json(f) + with open('examples/test.json', encoding='utf-8') as f: + table2 = Table.from_json(f) self.assertColumnNames(table2, self.column_names) self.assertColumnTypes(table2, [Number, Text, Boolean, Date, DateTime, TimeDelta]) diff --git a/tests/test_mapped_sequence.py b/tests/test_mapped_sequence.py index b113ecc6..b41c4e0e 100644 --- a/tests/test_mapped_sequence.py +++ b/tests/test_mapped_sequence.py @@ -1,11 +1,6 @@ #!/usr/bin/env python -try: - import unittest2 as unittest -except ImportError: - import unittest - -import six +import unittest from agate.mapped_sequence import MappedSequence @@ -24,20 +19,14 @@ def test_is_immutable(self): self.row['one'] = 100 def test_stringify(self): - if six.PY2: - self.assertEqual(str(self.row), "") - else: - self.assertEqual(str(self.row), "") + self.assertEqual(str(self.row), "") def test_stringify_long(self): column_names = ('one', 'two', 'three', 'four', 'five', 'six') data = (u'a', u'b', u'c', u'd', u'e', u'f') row = MappedSequence(data, column_names) - if six.PY2: - self.assertEqual(str(row), "") - else: - self.assertEqual(str(row), "") + self.assertEqual(str(row), "") def test_length(self): self.assertEqual(len(self.row), 3) diff --git a/tests/test_py3.py b/tests/test_py3.py index e7055154..0ff22634 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -4,19 +4,13 @@ import csv import os import platform - -try: - import unittest2 as unittest -except ImportError: - import unittest - -import six +import unittest +from io import StringIO from agate import csv_py3 from agate.exceptions import FieldSizeLimitError -@unittest.skipIf(six.PY2, "Not supported in Python 2.") class TestReader(unittest.TestCase): def setUp(self): self.rows = [ @@ -66,7 +60,6 @@ def test_line_numbers(self): self.assertEqual(a, b) -@unittest.skipIf(six.PY2, "Not supported in Python 2.") class TestFieldSizeLimit(unittest.TestCase): def setUp(self): self.lim = csv.field_size_limit() @@ -96,16 +89,15 @@ def test_field_size_limit(self): self.assertEqual(['a' * 10], c.__next__()) -@unittest.skipIf(six.PY2, "Not supported in Python 2.") class TestWriter(unittest.TestCase): def test_utf8(self): - output = six.StringIO() + output = StringIO() writer = csv_py3.Writer(output) writer.writerow(['a', 'b', 'c']) writer.writerow(['1', '2', '3']) writer.writerow(['4', '5', u'ʤ']) - written = six.StringIO(output.getvalue()) + written = StringIO(output.getvalue()) reader = csv_py3.Reader(written) self.assertEqual(next(reader), ['a', 'b', 'c']) @@ -113,13 +105,13 @@ def test_utf8(self): self.assertEqual(next(reader), ['4', '5', u'ʤ']) def test_writer_alias(self): - output = six.StringIO() + output = StringIO() writer = csv_py3.writer(output) writer.writerow(['a', 'b', 'c']) writer.writerow(['1', '2', '3']) writer.writerow(['4', '5', u'ʤ']) - written = six.StringIO(output.getvalue()) + written = StringIO(output.getvalue()) reader = csv_py3.reader(written) self.assertEqual(next(reader), ['a', 'b', 'c']) @@ -127,13 +119,13 @@ def test_writer_alias(self): self.assertEqual(next(reader), ['4', '5', u'ʤ']) def test_line_numbers(self): - output = six.StringIO() + output = StringIO() writer = csv_py3.Writer(output, line_numbers=True) writer.writerow(['a', 'b', 'c']) writer.writerow(['1', '2', '3']) writer.writerow(['4', '5', u'ʤ']) - written = six.StringIO(output.getvalue()) + written = StringIO(output.getvalue()) reader = csv_py3.Reader(written) self.assertEqual(next(reader), ['line_number', 'a', 'b', 'c']) @@ -141,7 +133,7 @@ def test_line_numbers(self): self.assertEqual(next(reader), ['2', '4', '5', u'ʤ']) def test_writerows(self): - output = six.StringIO() + output = StringIO() writer = csv_py3.Writer(output) writer.writerows([ ['a', 'b', 'c'], @@ -149,7 +141,7 @@ def test_writerows(self): ['4', '5', u'ʤ'] ]) - written = six.StringIO(output.getvalue()) + written = StringIO(output.getvalue()) reader = csv_py3.Reader(written) self.assertEqual(next(reader), ['a', 'b', 'c']) @@ -157,7 +149,6 @@ def test_writerows(self): self.assertEqual(next(reader), ['4', '5', u'ʤ']) -@unittest.skipIf(six.PY2, "Not supported in Python 2.") class TestDictReader(unittest.TestCase): def setUp(self): self.rows = [ @@ -183,10 +174,9 @@ def test_reader_alias(self): self.assertEqual(next(reader), dict(zip(self.rows[0], self.rows[1]))) -@unittest.skipIf(six.PY2, "Not supported in Python 2.") class TestDictWriter(unittest.TestCase): def setUp(self): - self.output = six.StringIO() + self.output = StringIO() def tearDown(self): self.output.close() @@ -244,8 +234,8 @@ def test_writerows(self): self.assertEqual(result, 'line_number,a,b,c\n1,1,2,☃\n') -@unittest.skipIf(six.PY2 or platform.system() == 'Linux' and six.PY3, - "Not supported in Python 2. Test inexplicably fails intermittently on Linux and Python 3.") +@unittest.skipIf(platform.system() == 'Linux', + "Test inexplicably fails intermittently on Linux.") class TestSniffer(unittest.TestCase): def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: diff --git a/tests/test_table/__init__.py b/tests/test_table/__init__.py index bd86a450..d4dfb577 100644 --- a/tests/test_table/__init__.py +++ b/tests/test_table/__init__.py @@ -1,14 +1,8 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal - import warnings - -import six +from decimal import Decimal from agate import Table from agate.computations import Formula @@ -303,24 +297,11 @@ def test_stringify(self): table = Table(self.rows, column_names) - if six.PY2: - u = unicode(table) # noqa: F821 - - self.assertIn('foo', u) - self.assertIn('bar', u) - self.assertIn(u'👍', u) - - s = str(table) - - self.assertIn('foo', s) - self.assertIn('bar', s) - self.assertIn(u'👍'.encode('utf-8'), s) - else: - u = str(table) + u = str(table) - self.assertIn('foo', u) - self.assertIn('bar', u) - self.assertIn(u'👍', u) + self.assertIn('foo', u) + self.assertIn('bar', u) + self.assertIn(u'👍', u) def test_str(self): warnings.simplefilter('ignore') diff --git a/tests/test_table/test_bins.py b/tests/test_table/test_bins.py index 2cf6d008..77729f55 100644 --- a/tests/test_table/test_bins.py +++ b/tests/test_table/test_bins.py @@ -1,12 +1,9 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -from babel.numbers import get_decimal_symbol +from decimal import Decimal -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal +from babel.numbers import get_decimal_symbol from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_table/test_compute.py b/tests/test_table/test_compute.py index e0bbca88..a18d7e48 100644 --- a/tests/test_table/test_compute.py +++ b/tests/test_table/test_compute.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import six from agate import Table from agate.computations import Formula @@ -45,7 +44,7 @@ def test_compute(self): def test_compute_multiple(self): new_table = self.table.compute([ ('number', Formula(self.number_type, lambda r: r['two'] + r['three'])), - ('text', Formula(self.text_type, lambda r: (r['one'] or '-') + six.text_type(r['three']))) + ('text', Formula(self.text_type, lambda r: (r['one'] or '-') + str(r['three']))) ]) self.assertIsNot(new_table, self.table) @@ -61,7 +60,7 @@ def test_compute_with_row_names(self): new_table = table.compute([ ('number', Formula(self.number_type, lambda r: r['two'] + r['three'])), - ('text', Formula(self.text_type, lambda r: (r['one'] or '-') + six.text_type(r['three']))) + ('text', Formula(self.text_type, lambda r: (r['one'] or '-') + str(r['three']))) ]) self.assertRowNames(new_table, [3, 5, 4, 6]) diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index fe6e2d0c..b4d4f220 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -4,8 +4,6 @@ import io import warnings -import six - from agate import Table from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.testcase import AgateTestCase @@ -58,10 +56,7 @@ def test_from_csv_cr(self): def test_from_csv_file_like_object(self): table1 = Table(self.rows, self.column_names, self.column_types) - if six.PY2: - f = open('examples/test.csv', 'rb') - else: - f = io.open('examples/test.csv', encoding='utf-8') + f = io.open('examples/test.csv', encoding='utf-8') table2 = Table.from_csv(f) f.close() diff --git a/tests/test_table/test_group_by.py b/tests/test_table/test_group_by.py index 73751b2a..05b9f12c 100644 --- a/tests/test_table/test_group_by.py +++ b/tests/test_table/test_group_by.py @@ -1,10 +1,7 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal +from decimal import Decimal from agate import Table, TableSet from agate.data_types import Boolean, Number, Text diff --git a/tests/test_table/test_homogenize.py b/tests/test_table/test_homogenize.py index 86ea1fd0..976f74db 100644 --- a/tests/test_table/test_homogenize.py +++ b/tests/test_table/test_homogenize.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -from six.moves import range - from agate import Table from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_pivot.py b/tests/test_table/test_pivot.py index 72ef4fcc..48de7ef8 100644 --- a/tests/test_table/test_pivot.py +++ b/tests/test_table/test_pivot.py @@ -2,11 +2,7 @@ # -*- coding: utf8 -*- import sys - -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal +from decimal import Decimal from agate import Table from agate.aggregations import Sum diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index 83b6557d..95b3e755 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import six +from io import StringIO + from babel.numbers import format_decimal from agate import Table @@ -32,14 +33,14 @@ def setUp(self): def test_print_bars(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_bars('three', 'one', output=output) lines = output.getvalue().split('\n') # noqa def test_print_bars_width(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_bars('three', 'one', width=40, output=output) lines = output.getvalue().split('\n') @@ -48,7 +49,7 @@ def test_print_bars_width(self): def test_print_bars_width_overlap(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_bars('three', 'one', width=20, output=output) lines = output.getvalue().split('\n') @@ -94,7 +95,7 @@ def test_print_bars_invalid_values(self): def test_print_bars_with_nulls(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_bars('three', 'two', width=20, printable=True, output=output) diff --git a/tests/test_table/test_print_html.py b/tests/test_table/test_print_html.py index 90d47daf..f1b9126d 100644 --- a/tests/test_table/test_print_html.py +++ b/tests/test_table/test_print_html.py @@ -2,16 +2,15 @@ # -*- coding: utf8 -*- import warnings - -import six -from six.moves import html_parser +from html.parser import HTMLParser +from io import StringIO from agate import Table from agate.data_types import Number, Text from agate.testcase import AgateTestCase -class TableHTMLParser(html_parser.HTMLParser): +class TableHTMLParser(HTMLParser): """ Parser for use in testing HTML rendering of tables. """ @@ -20,7 +19,7 @@ def __init__(self): warnings.simplefilter('ignore') try: - html_parser.HTMLParser.__init__(self) + HTMLParser.__init__(self) finally: warnings.resetwarnings() @@ -111,7 +110,7 @@ def setUp(self): def test_print_html(self): table = Table(self.rows, self.column_names, self.column_types) - table_html = six.StringIO() + table_html = StringIO() table.print_html(output=table_html) table_html = table_html.getvalue() @@ -139,7 +138,7 @@ def test_print_html(self): def test_print_html_tags(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_html(output=output) html = output.getvalue() @@ -150,7 +149,7 @@ def test_print_html_tags(self): def test_print_html_max_rows(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_html(max_rows=2, output=output) html = output.getvalue() @@ -161,7 +160,7 @@ def test_print_html_max_rows(self): def test_print_html_max_columns(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_html(max_columns=2, output=output) html = output.getvalue() diff --git a/tests/test_table/test_print_structure.py b/tests/test_table/test_print_structure.py index e60a84fe..7d742843 100644 --- a/tests/test_table/test_print_structure.py +++ b/tests/test_table/test_print_structure.py @@ -1,7 +1,7 @@ # !/usr/bin/env python # -*- coding: utf8 -*- -import six +from io import StringIO from agate import Table from agate.data_types import Number, Text @@ -30,7 +30,7 @@ def setUp(self): def test_print_structure(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_structure(output=output) lines = output.getvalue().strip().split('\n') diff --git a/tests/test_table/test_print_table.py b/tests/test_table/test_print_table.py index aebb0242..bd59c3d1 100644 --- a/tests/test_table/test_print_table.py +++ b/tests/test_table/test_print_table.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -import six +from io import StringIO + from babel.numbers import get_decimal_symbol from agate import Table @@ -33,7 +34,7 @@ def setUp(self): def test_print_table(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_table(output=output) lines = output.getvalue().split('\n') @@ -43,7 +44,7 @@ def test_print_table(self): def test_print_table_max_rows(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_table(max_rows=2, output=output) lines = output.getvalue().split('\n') @@ -53,7 +54,7 @@ def test_print_table_max_rows(self): def test_print_table_max_columns(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_table(max_columns=2, output=output) lines = output.getvalue().split('\n') @@ -75,7 +76,7 @@ def test_print_table_max_precision(self): ] table = Table(rows, column_names, column_types) - output = six.StringIO() + output = StringIO() table.print_table(output=output, max_precision=2) lines = output.getvalue().split('\n') @@ -102,7 +103,7 @@ def test_print_table_max_column_width(self): column_names = ['one', 'two', 'three', 'also, this is long'] table = Table(rows, column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_table(output=output, max_column_width=7) lines = output.getvalue().split('\n') @@ -117,7 +118,7 @@ def test_print_table_locale_american(self): """ table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_table(max_columns=2, output=output, locale='en_US') # If it's working, 2000 should appear as the english '2,000' self.assertTrue("2,000" in output.getvalue()) @@ -129,7 +130,7 @@ def test_print_table_locale_german(self): """ table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.print_table(max_columns=2, output=output, locale='de_DE.UTF-8') # If it's working, the english '2,000' should appear as '2.000' self.assertTrue("2.000" in output.getvalue()) diff --git a/tests/test_table/test_to_csv.py b/tests/test_table/test_to_csv.py index 917d40f9..e9853e41 100644 --- a/tests/test_table/test_to_csv.py +++ b/tests/test_table/test_to_csv.py @@ -3,8 +3,7 @@ import os import sys - -import six +from io import StringIO from agate import Table from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta @@ -64,7 +63,7 @@ def test_to_csv_file_like_object(self): def test_to_csv_to_stdout(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.to_csv(output) contents1 = output.getvalue() @@ -94,7 +93,7 @@ def test_print_csv(self): table = Table(self.rows, self.column_names, self.column_types) old = sys.stdout - sys.stdout = six.StringIO() + sys.stdout = StringIO() try: table.print_csv() diff --git a/tests/test_table/test_to_json.py b/tests/test_table/test_to_json.py index d98eb36e..c3da4805 100644 --- a/tests/test_table/test_to_json.py +++ b/tests/test_table/test_to_json.py @@ -4,8 +4,7 @@ import json import os import sys - -import six +from io import StringIO from agate import Table from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta @@ -31,7 +30,7 @@ def setUp(self): def test_to_json(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.to_json(output, indent=4) js1 = json.loads(output.getvalue()) @@ -44,7 +43,7 @@ def test_to_json(self): def test_to_json_key(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.to_json(output, key='text', indent=4) js1 = json.loads(output.getvalue()) @@ -57,7 +56,7 @@ def test_to_json_key(self): def test_to_json_non_string_key(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.to_json(output, key='number', indent=4) js1 = json.loads(output.getvalue()) @@ -70,7 +69,7 @@ def test_to_json_non_string_key(self): def test_to_json_key_func(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.to_json(output, key=lambda r: r['text'], indent=4) js1 = json.loads(output.getvalue()) @@ -83,7 +82,7 @@ def test_to_json_key_func(self): def test_to_json_newline_delimited(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() table.to_json(output, newline=True) js1 = json.loads(output.getvalue().split('\n')[0]) @@ -96,7 +95,7 @@ def test_to_json_newline_delimited(self): def test_to_json_error_newline_indent(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() with self.assertRaises(ValueError): table.to_json(output, newline=True, indent=4) @@ -104,7 +103,7 @@ def test_to_json_error_newline_indent(self): def test_to_json_error_newline_key(self): table = Table(self.rows, self.column_names, self.column_types) - output = six.StringIO() + output = StringIO() with self.assertRaises(ValueError): table.to_json(output, key='three', newline=True) @@ -144,7 +143,7 @@ def test_print_json(self): table = Table(self.rows, self.column_names, self.column_types) old = sys.stdout - sys.stdout = six.StringIO() + sys.stdout = StringIO() try: table.print_json() diff --git a/tests/test_tableset/__init__.py b/tests/test_tableset/__init__.py index 508bfcb2..e3b30715 100644 --- a/tests/test_tableset/__init__.py +++ b/tests/test_tableset/__init__.py @@ -1,14 +1,9 @@ #!/usr/bin/env python -from collections import OrderedDict - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - import json import shutil +from collections import OrderedDict +from io import StringIO from agate import Table, TableSet from agate.computations import Formula diff --git a/tests/test_tableset/test_aggregate.py b/tests/test_tableset/test_aggregate.py index b870d0d9..f423191f 100644 --- a/tests/test_tableset/test_aggregate.py +++ b/tests/test_tableset/test_aggregate.py @@ -1,11 +1,7 @@ #!/usr/bin/env python from collections import OrderedDict - -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal +from decimal import Decimal from agate import Table, TableSet from agate.aggregations import Count, MaxLength, Mean, Min, Sum diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 56d8f96e..196ca11b 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -1,10 +1,7 @@ #!/usr/bin/env python # -*- coding: utf8 -*- -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.type_tester import TypeTester diff --git a/tests/test_utils.py b/tests/test_utils.py index 9df54fcb..9dae3b6a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,14 +1,7 @@ #!/usr/bin/env python -try: - from cdecimal import Decimal -except ImportError: # pragma: no cover - from decimal import Decimal - -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest +from decimal import Decimal from agate.utils import Quantiles, letter_name, round_limits From 9abb42657ea824e7fdbc47bc7d8927ae0d5b8395 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:17:54 -0500 Subject: [PATCH 115/163] chore: Remove use of u'' --- agate/config.py | 32 ++++++++++----------- agate/data_types/number.py | 4 +-- agate/mapped_sequence.py | 9 +++--- agate/table/bins.py | 6 ++-- agate/table/print_bars.py | 8 +++--- agate/table/rename.py | 4 +-- agate/utils.py | 6 ++-- docs/conf.py | 4 +-- tests/test_aggregations.py | 2 +- tests/test_data_types.py | 20 ++++++------- tests/test_from_json.py | 4 +-- tests/test_mapped_sequence.py | 20 ++++++------- tests/test_py3.py | 42 ++++++++++++++-------------- tests/test_table/__init__.py | 34 +++++++++++----------- tests/test_table/test_aggregate.py | 2 +- tests/test_table/test_bins.py | 12 ++++---- tests/test_table/test_charting.py | 2 +- tests/test_table/test_from_csv.py | 2 +- tests/test_table/test_order_py.py | 4 +-- tests/test_table/test_print_bars.py | 4 +-- tests/test_table/test_print_html.py | 2 +- tests/test_table/test_print_table.py | 18 ++++++------ tests/test_table/test_rename.py | 4 +-- tests/test_table/test_to_csv.py | 2 +- tests/test_table/test_to_json.py | 2 +- tests/test_type_tester.py | 4 +-- 26 files changed, 126 insertions(+), 127 deletions(-) diff --git a/agate/config.py b/agate/config.py index fc112ec4..c28ef6c5 100644 --- a/agate/config.py +++ b/agate/config.py @@ -13,21 +13,21 @@ +=========================+==========================================+=========================================+ | default_locale | Default locale for number formatting | default_locale('LC_NUMERIC') or 'en_US' | +-------------------------+------------------------------------------+-----------------------------------------+ -| horizontal_line_char | Character to render for horizontal lines | u'-' | +| horizontal_line_char | Character to render for horizontal lines | '-' | +-------------------------+------------------------------------------+-----------------------------------------+ -| vertical_line_char | Character to render for vertical lines | u'|' | +| vertical_line_char | Character to render for vertical lines | '|' | +-------------------------+------------------------------------------+-----------------------------------------+ -| bar_char | Character to render for bar chart units | u'░' | +| bar_char | Character to render for bar chart units | '░' | +-------------------------+------------------------------------------+-----------------------------------------+ -| printable_bar_char | Printable character for bar chart units | u':' | +| printable_bar_char | Printable character for bar chart units | ':' | +-------------------------+------------------------------------------+-----------------------------------------+ -| zero_line_char | Character to render for zero line units | u'▓' | +| zero_line_char | Character to render for zero line units | '▓' | +-------------------------+------------------------------------------+-----------------------------------------+ -| printable_zero_line_char| Printable character for zero line units | u'|' | +| printable_zero_line_char| Printable character for zero line units | '|' | +-------------------------+------------------------------------------+-----------------------------------------+ -| tick_char | Character to render for axis ticks | u'+' | +| tick_char | Character to render for axis ticks | '+' | +-------------------------+------------------------------------------+-----------------------------------------+ -| ellipsis_chars | Characters to render for ellipsis | u'...' | +| ellipsis_chars | Characters to render for ellipsis | '...' | +-------------------------+------------------------------------------+-----------------------------------------+ """ @@ -38,21 +38,21 @@ #: Default locale for number formatting 'default_locale': default_locale('LC_NUMERIC') or 'en_US', #: Character to render for horizontal lines - 'horizontal_line_char': u'-', + 'horizontal_line_char': '-', #: Character to render for vertical lines - 'vertical_line_char': u'|', + 'vertical_line_char': '|', #: Character to render for bar chart units - 'bar_char': u'░', + 'bar_char': '░', #: Printable character to render for bar chart units - 'printable_bar_char': u':', + 'printable_bar_char': ':', #: Character to render for zero line units - 'zero_line_char': u'▓', + 'zero_line_char': '▓', #: Printable character to render for zero line units - 'printable_zero_line_char': u'|', + 'printable_zero_line_char': '|', #: Character to render for axis ticks - 'tick_char': u'+', + 'tick_char': '+', #: Characters to render for ellipsis - 'ellipsis_chars': u'...', + 'ellipsis_chars': '...', } diff --git a/agate/data_types/number.py b/agate/data_types/number.py index e41e8fa4..2d5a7786 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -10,8 +10,8 @@ from agate.exceptions import CastError #: A list of currency symbols sourced from `Xe `_. -DEFAULT_CURRENCY_SYMBOLS = [u'؋', u'$', u'ƒ', u'៛', u'¥', u'₡', u'₱', u'£', u'€', u'¢', u'﷼', u'₪', u'₩', u'₭', u'₮', - u'₦', u'฿', u'₤', u'₫'] +DEFAULT_CURRENCY_SYMBOLS = ['؋', '$', 'ƒ', '៛', '¥', '₡', '₱', '£', '€', '¢', '﷼', '₪', '₩', '₭', '₮', + '₦', '฿', '₤', '₫'] POSITIVE = Decimal('1') NEGATIVE = Decimal('-1') diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index 3b4c3cd0..2ca22995 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -60,12 +60,12 @@ def __unicode__(self): """ Print a unicode sample of the contents of this sequence. """ - sample = u', '.join(repr(d) for d in self.values()[:5]) + sample = ', '.join(repr(d) for d in self.values()[:5]) if len(self) > 5: - sample = u'%s, ...' % sample + sample = '%s, ...' % sample - return u'' % (type(self).__name__, sample) + return '' % (type(self).__name__, sample) def __str__(self): """ @@ -153,8 +153,7 @@ def get(self, key, default=None): except KeyError: if default: return default - else: - return None + return None @memoize def dict(self): diff --git a/agate/table/bins.py b/agate/table/bins.py index ee14d81c..e05e935d 100644 --- a/agate/table/bins.py +++ b/agate/table/bins.py @@ -62,9 +62,9 @@ def name_bin(i, j, first_exclusive=True, last_exclusive=False): inclusive = format_decimal(i, format=break_formatter) exclusive = format_decimal(j, format=break_formatter) - output = u'[' if first_exclusive else u'(' - output += u'%s - %s' % (inclusive, exclusive) - output += u']' if last_exclusive else u')' + output = '[' if first_exclusive else '(' + output += '%s - %s' % (inclusive, exclusive) + output += ']' if last_exclusive else ')' return output diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index 739fc391..2dba87de 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -177,7 +177,7 @@ def write(line): output.write(line + '\n') # Chart top - top_line = u'%s %s' % (y_label.ljust(max_label_width), x_label.rjust(max_value_width)) + top_line = '%s %s' % (y_label.ljust(max_label_width), x_label.rjust(max_value_width)) write(top_line) # Bars @@ -196,7 +196,7 @@ def write(line): bar = bar_mark * bar_width if value is not None and value >= 0: - gap = (u' ' * plot_negative_width) + gap = (' ' * plot_negative_width) # All positive if x_min <= 0: @@ -204,7 +204,7 @@ def write(line): else: bar = bar + gap + zero_mark else: - bar = u' ' * (plot_negative_width - bar_width) + bar + bar = ' ' * (plot_negative_width - bar_width) + bar # All negative or mixed signs if value is None or x_max > value: @@ -216,7 +216,7 @@ def write(line): # Axis & ticks axis = horizontal_line * plot_width - tick_text = u' ' * width + tick_text = ' ' * width for i, (tick, label) in enumerate(ticks_formatted.items()): # First tick diff --git a/agate/table/rename.py b/agate/table/rename.py index e023e351..14069ce3 100644 --- a/agate/table/rename.py +++ b/agate/table/rename.py @@ -57,5 +57,5 @@ def rename(self, column_names=None, row_names=None, slug_columns=False, slug_row row_names = self._row_names return Table(self._rows, column_names, self._column_types, row_names=row_names, _is_fork=False) - else: - return self._fork(self._rows, column_names, self._column_types, row_names=row_names) + + return self._fork(self._rows, column_names, self._column_types, row_names=row_names) diff --git a/agate/utils.py b/agate/utils.py index 8171c529..a51fe5f8 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -163,9 +163,9 @@ def make_number_formatter(decimal_places, add_ellipsis=False): :param add_ellipsis: Optionally add an ellipsis symbol at the end of a number """ - fraction = u'0' * decimal_places - ellipsis = u'…' if add_ellipsis else u'' - return u''.join([u'#,##0.', fraction, ellipsis, u';-#,##0.', fraction, ellipsis]) + fraction = '0' * decimal_places + ellipsis = '…' if add_ellipsis else '' + return ''.join(['#,##0.', fraction, ellipsis, ';-#,##0.', fraction, ellipsis]) def round_limits(minimum, maximum): diff --git a/docs/conf.py b/docs/conf.py index 4153a3c8..9b9bdd0b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,8 +26,8 @@ master_doc = 'index' # Metadata -project = u'agate' -copyright = u'2017, Christopher Groskopf' +project = 'agate' +copyright = '2017, Christopher Groskopf' version = '1.6.3' release = version diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 5ce4c659..239b7ffb 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -773,7 +773,7 @@ def test_max_length_unicode(self): """ rows = [ ['a'], - [u'👍'], + ['👍'], ['w'] ] diff --git a/tests/test_data_types.py b/tests/test_data_types.py index 0f65571a..6e76b749 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -35,7 +35,7 @@ def test_test(self): self.assertEqual(self.type.test(datetime.timedelta(hours=4, minutes=10)), True) self.assertEqual(self.type.test('a'), True) self.assertEqual(self.type.test('A\nB'), True) - self.assertEqual(self.type.test(u'👍'), True) + self.assertEqual(self.type.test('👍'), True) self.assertEqual(self.type.test('05_leslie3d_base'), True) self.assertEqual(self.type.test('2016-12-29'), True) self.assertEqual(self.type.test('2016-12-29T11:43:30Z'), True) @@ -43,9 +43,9 @@ def test_test(self): self.assertEqual(self.type.test('2016-12-29T11:43:30-06:00'), True) def test_cast(self): - values = ('a', 1, None, Decimal('2.7'), 'n/a', u'👍', ' foo', 'foo ') + values = ('a', 1, None, Decimal('2.7'), 'n/a', '👍', ' foo', 'foo ') casted = tuple(self.type.cast(v) for v in values) - self.assertSequenceEqual(casted, ('a', '1', None, '2.7', None, u'👍', ' foo', 'foo ')) + self.assertSequenceEqual(casted, ('a', '1', None, '2.7', None, '👍', ' foo', 'foo ')) def test_no_cast_nulls(self): values = ('', 'N/A', None) @@ -85,7 +85,7 @@ def test_test(self): self.assertEqual(self.type.test(datetime.timedelta(hours=4, minutes=10)), False) self.assertEqual(self.type.test('a'), False) self.assertEqual(self.type.test('A\nB'), False) - self.assertEqual(self.type.test(u'👍'), False) + self.assertEqual(self.type.test('👍'), False) self.assertEqual(self.type.test('05_leslie3d_base'), False) self.assertEqual(self.type.test('2016-12-29'), False) self.assertEqual(self.type.test('2016-12-29T11:43:30Z'), False) @@ -134,7 +134,7 @@ def test_test(self): self.assertEqual(self.type.test(datetime.timedelta(hours=4, minutes=10)), False) self.assertEqual(self.type.test('a'), False) self.assertEqual(self.type.test('A\nB'), False) - self.assertEqual(self.type.test(u'👍'), False) + self.assertEqual(self.type.test('👍'), False) self.assertEqual(self.type.test('05_leslie3d_base'), False) self.assertEqual(self.type.test('2016-12-29'), False) self.assertEqual(self.type.test('2016-12-29T11:43:30Z'), False) @@ -155,7 +155,7 @@ def test_boolean_cast(self): self.assertSequenceEqual(casted, (Decimal('1'), Decimal('0'))) def test_currency_cast(self): - values = ('$2.70', '-$0.70', u'€14', u'50¢', u'-75¢', u'-$1,287') + values = ('$2.70', '-$0.70', '€14', '50¢', '-75¢', '-$1,287') casted = tuple(self.type.cast(v) for v in values) self.assertSequenceEqual( casted, @@ -205,7 +205,7 @@ def test_test(self): self.assertEqual(self.type.test(datetime.timedelta(hours=4, minutes=10)), False) self.assertEqual(self.type.test('a'), False) self.assertEqual(self.type.test('A\nB'), False) - self.assertEqual(self.type.test(u'👍'), False) + self.assertEqual(self.type.test('👍'), False) self.assertEqual(self.type.test('05_leslie3d_base'), False) self.assertEqual(self.type.test('2016-12-29'), True) self.assertEqual(self.type.test('2016-12-29T11:43:30Z'), False) @@ -271,7 +271,7 @@ def test_cast_format_locale(self): def test_cast_locale(self): date_type = Date(locale='fr_FR') - values = ('01 mars 1994', u'jeudi 17 février 2011', None, '5 janvier 1984', 'n/a') + values = ('01 mars 1994', 'jeudi 17 février 2011', None, '5 janvier 1984', 'n/a') casted = tuple(date_type.cast(v) for v in values) self.assertSequenceEqual(casted, ( datetime.date(1994, 3, 1), @@ -315,7 +315,7 @@ def test_test(self): self.assertEqual(self.type.test(datetime.timedelta(hours=4, minutes=10)), False) self.assertEqual(self.type.test('a'), False) self.assertEqual(self.type.test('A\nB'), False) - self.assertEqual(self.type.test(u'👍'), False) + self.assertEqual(self.type.test('👍'), False) self.assertEqual(self.type.test('05_leslie3d_base'), False) self.assertEqual(self.type.test('2016-12-29'), True) self.assertEqual(self.type.test('2016-12-29T11:43:30Z'), True) @@ -455,7 +455,7 @@ def test_test(self): self.assertEqual(self.type.test(datetime.timedelta(hours=4, minutes=10)), True) self.assertEqual(self.type.test('a'), False) self.assertEqual(self.type.test('A\nB'), False) - self.assertEqual(self.type.test(u'👍'), False) + self.assertEqual(self.type.test('👍'), False) self.assertEqual(self.type.test('05_leslie3d_base'), False) self.assertEqual(self.type.test('2016-12-29'), False) self.assertEqual(self.type.test('2016-12-29T11:43:30Z'), False) diff --git a/tests/test_from_json.py b/tests/test_from_json.py index ea191db6..d20aa30a 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -12,7 +12,7 @@ class TestJSON(AgateTestCase): def setUp(self): self.rows = ( (1, 'a', True, '11/4/2015', '11/4/2015 12:22 PM', '4:15'), - (2, u'👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), + (2, '👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), (None, 'b', None, None, None, None) ) @@ -58,7 +58,7 @@ def test_from_json_mixed_keys(self): self.assertRows(table, [ [1, 4, 'a', None, None], [2, 3, 'b', 'd', None], - [None, 2, u'👍', None, 5] + [None, 2, '👍', None, 5] ]) def test_from_json_nested(self): diff --git a/tests/test_mapped_sequence.py b/tests/test_mapped_sequence.py index b41c4e0e..b2ec1d3f 100644 --- a/tests/test_mapped_sequence.py +++ b/tests/test_mapped_sequence.py @@ -8,7 +8,7 @@ class TestMappedSequence(unittest.TestCase): def setUp(self): self.column_names = ('one', 'two', 'three') - self.data = (u'a', u'b', u'c') + self.data = ('a', 'b', 'c') self.row = MappedSequence(self.data, self.column_names) def test_is_immutable(self): @@ -23,7 +23,7 @@ def test_stringify(self): def test_stringify_long(self): column_names = ('one', 'two', 'three', 'four', 'five', 'six') - data = (u'a', u'b', u'c', u'd', u'e', u'f') + data = ('a', 'b', 'c', 'd', 'e', 'f') row = MappedSequence(data, column_names) self.assertEqual(str(row), "") @@ -34,19 +34,19 @@ def test_length(self): def test_eq(self): row2 = MappedSequence(self.data, self.column_names) - self.assertTrue(self.row == (u'a', u'b', u'c')) - self.assertTrue(self.row == [u'a', u'b', u'c']) + self.assertTrue(self.row == ('a', 'b', 'c')) + self.assertTrue(self.row == ['a', 'b', 'c']) self.assertTrue(self.row == row2) - self.assertFalse(self.row == (u'a', u'b', u'c', u'd')) + self.assertFalse(self.row == ('a', 'b', 'c', 'd')) self.assertFalse(self.row == 1) def test_ne(self): row2 = MappedSequence(self.data, self.column_names) - self.assertFalse(self.row != (u'a', u'b', u'c')) - self.assertFalse(self.row != [u'a', u'b', u'c']) + self.assertFalse(self.row != ('a', 'b', 'c')) + self.assertFalse(self.row != ['a', 'b', 'c']) self.assertFalse(self.row != row2) - self.assertTrue(self.row != (u'a', u'b', u'c', u'd')) + self.assertTrue(self.row != ('a', 'b', 'c', 'd')) self.assertTrue(self.row != 1) def test_contains(self): @@ -55,10 +55,10 @@ def test_contains(self): def test_set_item(self): with self.assertRaises(TypeError): - self.row['one'] = u't' + self.row['one'] = 't' with self.assertRaises(TypeError): - self.row['five'] = u'g' + self.row['five'] = 'g' def test_get_item(self): self.assertEqual(self.row['one'], 'a') diff --git a/tests/test_py3.py b/tests/test_py3.py index 0ff22634..24b852d0 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -52,7 +52,7 @@ def test_line_numbers(self): sample_rows = [ ['line_numbers', 'number', 'text', 'boolean', 'date', 'datetime', 'timedelta'], ['1', '1', 'a', 'True', '2015-11-04', '2015-11-04T12:22:00', '0:04:15'], - ['2', '2', u'👍', 'False', '2015-11-05', '2015-11-04T12:45:00', '0:06:18'], + ['2', '2', '👍', 'False', '2015-11-05', '2015-11-04T12:45:00', '0:06:18'], ['3', '', 'b', '', '', '', ''] ] @@ -95,42 +95,42 @@ def test_utf8(self): writer = csv_py3.Writer(output) writer.writerow(['a', 'b', 'c']) writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) + writer.writerow(['4', '5', 'ʤ']) written = StringIO(output.getvalue()) reader = csv_py3.Reader(written) self.assertEqual(next(reader), ['a', 'b', 'c']) self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) + self.assertEqual(next(reader), ['4', '5', 'ʤ']) def test_writer_alias(self): output = StringIO() writer = csv_py3.writer(output) writer.writerow(['a', 'b', 'c']) writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) + writer.writerow(['4', '5', 'ʤ']) written = StringIO(output.getvalue()) reader = csv_py3.reader(written) self.assertEqual(next(reader), ['a', 'b', 'c']) self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) + self.assertEqual(next(reader), ['4', '5', 'ʤ']) def test_line_numbers(self): output = StringIO() writer = csv_py3.Writer(output, line_numbers=True) writer.writerow(['a', 'b', 'c']) writer.writerow(['1', '2', '3']) - writer.writerow(['4', '5', u'ʤ']) + writer.writerow(['4', '5', 'ʤ']) written = StringIO(output.getvalue()) reader = csv_py3.Reader(written) self.assertEqual(next(reader), ['line_number', 'a', 'b', 'c']) self.assertEqual(next(reader), ['1', '1', '2', '3']) - self.assertEqual(next(reader), ['2', '4', '5', u'ʤ']) + self.assertEqual(next(reader), ['2', '4', '5', 'ʤ']) def test_writerows(self): output = StringIO() @@ -138,7 +138,7 @@ def test_writerows(self): writer.writerows([ ['a', 'b', 'c'], ['1', '2', '3'], - ['4', '5', u'ʤ'] + ['4', '5', 'ʤ'] ]) written = StringIO(output.getvalue()) @@ -146,7 +146,7 @@ def test_writerows(self): reader = csv_py3.Reader(written) self.assertEqual(next(reader), ['a', 'b', 'c']) self.assertEqual(next(reader), ['1', '2', '3']) - self.assertEqual(next(reader), ['4', '5', u'ʤ']) + self.assertEqual(next(reader), ['4', '5', 'ʤ']) class TestDictReader(unittest.TestCase): @@ -185,9 +185,9 @@ def test_writer(self): writer = csv_py3.DictWriter(self.output, ['a', 'b', 'c']) writer.writeheader() writer.writerow({ - u'a': u'1', - u'b': u'2', - u'c': u'☃' + 'a': '1', + 'b': '2', + 'c': '☃' }) result = self.output.getvalue() @@ -198,9 +198,9 @@ def test_writer_alias(self): writer = csv_py3.DictWriter(self.output, ['a', 'b', 'c']) writer.writeheader() writer.writerow({ - u'a': u'1', - u'b': u'2', - u'c': u'☃' + 'a': '1', + 'b': '2', + 'c': '☃' }) result = self.output.getvalue() @@ -211,9 +211,9 @@ def test_line_numbers(self): writer = csv_py3.DictWriter(self.output, ['a', 'b', 'c'], line_numbers=True) writer.writeheader() writer.writerow({ - u'a': u'1', - u'b': u'2', - u'c': u'☃' + 'a': '1', + 'b': '2', + 'c': '☃' }) result = self.output.getvalue() @@ -224,9 +224,9 @@ def test_writerows(self): writer = csv_py3.DictWriter(self.output, ['a', 'b', 'c'], line_numbers=True) writer.writeheader() writer.writerows([{ - u'a': u'1', - u'b': u'2', - u'c': u'☃' + 'a': '1', + 'b': '2', + 'c': '☃' }]) result = self.output.getvalue() diff --git a/tests/test_table/__init__.py b/tests/test_table/__init__.py index d4dfb577..882179d8 100644 --- a/tests/test_table/__init__.py +++ b/tests/test_table/__init__.py @@ -17,7 +17,7 @@ def setUp(self): self.rows = ( (1, 4, 'a'), (2, 3, 'b'), - (None, 2, u'👍') + (None, 2, '👍') ) self.number_type = Number() @@ -69,7 +69,7 @@ def test_create_table_column_types(self): self.assertRows(table, [ (1, '4', 'a'), (2, '3', 'b'), - (None, '2', u'👍') + (None, '2', '👍') ]) def test_create_table_column_types_dict(self): @@ -226,8 +226,8 @@ def test_create_table_no_column_names(self): with self.assertRaises(KeyError): table.columns['one'] - self.assertSequenceEqual(table.columns[2], ('a', 'b', u'👍')) - self.assertSequenceEqual(table.columns['c'], ('a', 'b', u'👍')) + self.assertSequenceEqual(table.columns[2], ('a', 'b', '👍')) + self.assertSequenceEqual(table.columns['c'], ('a', 'b', '👍')) with self.assertRaises(KeyError): table.columns[''] @@ -252,7 +252,7 @@ def test_row_too_long(self): def test_row_names(self): table = Table(self.rows, self.column_names, self.column_types, row_names='three') - self.assertRowNames(table, ['a', 'b', u'👍']) + self.assertRowNames(table, ['a', 'b', '👍']) def test_row_names_non_string(self): table = Table(self.rows, self.column_names, self.column_types, row_names=[Decimal('2'), True, None]) @@ -264,7 +264,7 @@ def test_row_names_non_string(self): ]) self.assertSequenceEqual(table.rows[Decimal('2')], (1, 4, 'a')) self.assertSequenceEqual(table.rows[True], (2, 3, 'b')) - self.assertSequenceEqual(table.rows[None], (None, 2, u'👍')) + self.assertSequenceEqual(table.rows[None], (None, 2, '👍')) def test_row_names_int(self): with self.assertRaises(ValueError): @@ -276,11 +276,11 @@ def test_row_names_func(self): self.assertSequenceEqual(table.row_names, [ (Decimal('1'), 'a'), (Decimal('2'), 'b'), - (None, u'👍') + (None, '👍') ]) self.assertSequenceEqual(table.rows[(Decimal('1'), 'a')], (1, 4, 'a')) self.assertSequenceEqual(table.rows[(Decimal('2'), 'b')], (2, 3, 'b')) - self.assertSequenceEqual(table.rows[(None, u'👍')], (None, 2, u'👍')) + self.assertSequenceEqual(table.rows[(None, '👍')], (None, 2, '👍')) def test_row_names_invalid(self): @@ -293,7 +293,7 @@ def test_row_names_invalid(self): ) def test_stringify(self): - column_names = ['foo', 'bar', u'👍'] + column_names = ['foo', 'bar', '👍'] table = Table(self.rows, column_names) @@ -301,7 +301,7 @@ def test_stringify(self): self.assertIn('foo', u) self.assertIn('bar', u) - self.assertIn(u'👍', u) + self.assertIn('👍', u) def test_str(self): warnings.simplefilter('ignore') @@ -360,7 +360,7 @@ def test_select(self): self.assertRows(new_table, [ [4, 'a'], [3, 'b'], - [2, u'👍'] + [2, '👍'] ]) def test_select_single(self): @@ -372,14 +372,14 @@ def test_select_single(self): self.assertRows(new_table, [ ['a'], ['b'], - [u'👍'] + ['👍'] ]) def test_select_with_row_names(self): table = Table(self.rows, self.column_names, self.column_types, row_names='three') new_table = table.select(('three',)) - self.assertRowNames(new_table, ['a', 'b', u'👍']) + self.assertRowNames(new_table, ['a', 'b', '👍']) def test_select_does_not_exist(self): table = Table(self.rows, self.column_names, self.column_types) @@ -399,7 +399,7 @@ def test_exclude(self): self.assertRows(new_table, [ ['a'], ['b'], - [u'👍'] + ['👍'] ]) def test_exclude_single(self): @@ -414,14 +414,14 @@ def test_exclude_single(self): self.assertRows(new_table, [ [4, 'a'], [3, 'b'], - [2, u'👍'] + [2, '👍'] ]) def test_exclude_with_row_names(self): table = Table(self.rows, self.column_names, self.column_types, row_names='three') new_table = table.exclude(('one', 'two')) - self.assertRowNames(new_table, ['a', 'b', u'👍']) + self.assertRowNames(new_table, ['a', 'b', '👍']) def test_where(self): table = Table(self.rows, self.column_names, self.column_types) @@ -441,7 +441,7 @@ def test_where_with_row_names(self): table = Table(self.rows, self.column_names, self.column_types, row_names='three') new_table = table.where(lambda r: r['one'] in (2, None)) - self.assertRowNames(new_table, ['b', u'👍']) + self.assertRowNames(new_table, ['b', '👍']) def test_find(self): table = Table(self.rows, self.column_names, self.column_types) diff --git a/tests/test_table/test_aggregate.py b/tests/test_table/test_aggregate.py index a7ad6d0b..4ded61f5 100644 --- a/tests/test_table/test_aggregate.py +++ b/tests/test_table/test_aggregate.py @@ -12,7 +12,7 @@ def setUp(self): self.rows = ( (1, 4, 'a'), (2, 3, 'b'), - (None, 2, u'👍') + (None, 2, '👍') ) self.number_type = Number() diff --git a/tests/test_table/test_bins.py b/tests/test_table/test_bins.py index 77729f55..c60f3e81 100644 --- a/tests/test_table/test_bins.py +++ b/tests/test_table/test_bins.py @@ -111,15 +111,15 @@ def test_bins_decimals(self): self.assertSequenceEqual( new_table.rows[0], - [u'[0' + get_decimal_symbol() + u'0 - 0' + get_decimal_symbol() + u'1)', 10] + ['[0' + get_decimal_symbol() + '0 - 0' + get_decimal_symbol() + '1)', 10] ) self.assertSequenceEqual( new_table.rows[3], - [u'[0' + get_decimal_symbol() + u'3 - 0' + get_decimal_symbol() + u'4)', 10] + ['[0' + get_decimal_symbol() + '3 - 0' + get_decimal_symbol() + '4)', 10] ) self.assertSequenceEqual( new_table.rows[9], - [u'[0' + get_decimal_symbol() + u'9 - 1' + get_decimal_symbol() + u'0]', 10] + ['[0' + get_decimal_symbol() + '9 - 1' + get_decimal_symbol() + '0]', 10] ) def test_bins_nulls(self): @@ -137,14 +137,14 @@ def test_bins_nulls(self): self.assertSequenceEqual( new_table.rows[0], - [u'[0' + get_decimal_symbol() + u'0 - 0' + get_decimal_symbol() + u'1)', 10] + ['[0' + get_decimal_symbol() + '0 - 0' + get_decimal_symbol() + '1)', 10] ) self.assertSequenceEqual( new_table.rows[3], - [u'[0' + get_decimal_symbol() + u'3 - 0' + get_decimal_symbol() + u'4)', 10] + ['[0' + get_decimal_symbol() + '3 - 0' + get_decimal_symbol() + '4)', 10] ) self.assertSequenceEqual( new_table.rows[9], - [u'[0' + get_decimal_symbol() + u'9 - 1' + get_decimal_symbol() + u'0]', 10] + ['[0' + get_decimal_symbol() + '9 - 1' + get_decimal_symbol() + '0]', 10] ) self.assertSequenceEqual(new_table.rows[10], [None, 1]) diff --git a/tests/test_table/test_charting.py b/tests/test_table/test_charting.py index 1a543384..d9ea7d08 100644 --- a/tests/test_table/test_charting.py +++ b/tests/test_table/test_charting.py @@ -12,7 +12,7 @@ def setUp(self): self.rows = ( (1, 4, 'a'), (2, 3, 'b'), - (None, 2, u'👍') + (None, 2, '👍') ) self.number_type = Number() diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index b4d4f220..367e1634 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -14,7 +14,7 @@ class TestFromCSV(AgateTestCase): def setUp(self): self.rows = ( (1, 'a', True, '11/4/2015', '11/4/2015 12:22 PM', '4:15'), - (2, u'👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), + (2, '👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), (None, 'b', None, None, None, None) ) diff --git a/tests/test_table/test_order_py.py b/tests/test_table/test_order_py.py index 6a3cd23b..3f29fbe0 100644 --- a/tests/test_table/test_order_py.py +++ b/tests/test_table/test_order_py.py @@ -11,7 +11,7 @@ def setUp(self): self.rows = ( (1, 4, 'a'), (2, 3, 'b'), - (None, 2, u'👍') + (None, 2, '👍') ) self.number_type = Number() @@ -142,7 +142,7 @@ def test_order_by_with_row_names(self): table = Table(self.rows, self.column_names, self.column_types, row_names='three') new_table = table.order_by('two') - self.assertRowNames(new_table, [u'👍', 'b', 'a']) + self.assertRowNames(new_table, ['👍', 'b', 'a']) def test_order_by_empty_table(self): table = Table([], self.column_names) diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index 95b3e755..5c473dd6 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -100,8 +100,8 @@ def test_print_bars_with_nulls(self): output=output) self.assertEqual(output.getvalue(), "three two\n" - "a " + format_decimal(2000, format=u'#,##0') + " |:::::::\n" + "a " + format_decimal(2000, format='#,##0') + " |:::::::\n" "None - | \n" "c 1 | \n" " +------+\n" - " 0 " + format_decimal(2000, format=u'#,##0') + "\n") + " 0 " + format_decimal(2000, format='#,##0') + "\n") diff --git a/tests/test_table/test_print_html.py b/tests/test_table/test_print_html.py index f1b9126d..d1b67057 100644 --- a/tests/test_table/test_print_html.py +++ b/tests/test_table/test_print_html.py @@ -99,7 +99,7 @@ def setUp(self): self.rows = ( (1, 4, 'a'), (2, 3, 'b'), - (None, 2, u'👍') + (None, 2, '👍') ) self.number_type = Number() diff --git a/tests/test_table/test_print_table.py b/tests/test_table/test_print_table.py index bd59c3d1..187b41c0 100644 --- a/tests/test_table/test_print_table.py +++ b/tests/test_table/test_print_table.py @@ -81,17 +81,17 @@ def test_print_table_max_precision(self): lines = output.getvalue().split('\n') # Text shouldn't be affected - self.assertIn(u' 1.745 ', lines[2]) - self.assertIn(u' 11.123456 ', lines[3]) - self.assertIn(u' 0 ', lines[4]) + self.assertIn(' 1.745 ', lines[2]) + self.assertIn(' 11.123456 ', lines[3]) + self.assertIn(' 0 ', lines[4]) # Test real precision above max - self.assertIn(u' 1' + get_decimal_symbol() + u'74… ', lines[2]) - self.assertIn(u' 11' + get_decimal_symbol() + u'12… ', lines[3]) - self.assertIn(u' 0' + get_decimal_symbol() + u'00… ', lines[4]) + self.assertIn(' 1' + get_decimal_symbol() + '74… ', lines[2]) + self.assertIn(' 11' + get_decimal_symbol() + '12… ', lines[3]) + self.assertIn(' 0' + get_decimal_symbol() + '00… ', lines[4]) # Test real precision below max - self.assertIn(u' 1' + get_decimal_symbol() + u'72 ', lines[2]) - self.assertIn(u' 5' + get_decimal_symbol() + u'10 ', lines[3]) - self.assertIn(u' 0' + get_decimal_symbol() + u'10 ', lines[4]) + self.assertIn(' 1' + get_decimal_symbol() + '72 ', lines[2]) + self.assertIn(' 5' + get_decimal_symbol() + '10 ', lines[3]) + self.assertIn(' 0' + get_decimal_symbol() + '10 ', lines[4]) def test_print_table_max_column_width(self): rows = ( diff --git a/tests/test_table/test_rename.py b/tests/test_table/test_rename.py index 9ea5fd4e..c5740d80 100644 --- a/tests/test_table/test_rename.py +++ b/tests/test_table/test_rename.py @@ -93,7 +93,7 @@ def test_rename_slugify_rows(self): self.assertRowNames(table3, ['test.koz', 'test.2', 'test.2.2']) def test_rename_slugify_columns_in_place(self): - column_names = [u'Test kož', 'test 2', 'test 2'] + column_names = ['Test kož', 'test 2', 'test 2'] warnings.simplefilter('ignore') @@ -105,7 +105,7 @@ def test_rename_slugify_columns_in_place(self): table2 = table.rename(slug_columns=True) table3 = table.rename(slug_columns=True, separator='.') - self.assertColumnNames(table, [u'Test kož', 'test 2', 'test 2_2']) + self.assertColumnNames(table, ['Test kož', 'test 2', 'test 2_2']) self.assertColumnNames(table2, ['test_koz', 'test_2', 'test_2_2']) self.assertColumnNames(table3, ['test.koz', 'test.2', 'test.2.2']) diff --git a/tests/test_table/test_to_csv.py b/tests/test_table/test_to_csv.py index e9853e41..525d336c 100644 --- a/tests/test_table/test_to_csv.py +++ b/tests/test_table/test_to_csv.py @@ -14,7 +14,7 @@ class TestToCSV(AgateTestCase): def setUp(self): self.rows = ( (1, 'a', True, '11/4/2015', '11/4/2015 12:22 PM', '4:15'), - (2, u'👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), + (2, '👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), (None, 'b', None, None, None, None) ) diff --git a/tests/test_table/test_to_json.py b/tests/test_table/test_to_json.py index c3da4805..21af2515 100644 --- a/tests/test_table/test_to_json.py +++ b/tests/test_table/test_to_json.py @@ -15,7 +15,7 @@ class TestJSON(AgateTestCase): def setUp(self): self.rows = ( (1, 'a', True, '11/4/2015', '11/4/2015 12:22 PM', '4:15'), - (2, u'👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), + (2, '👍', False, '11/5/2015', '11/4/2015 12:45 PM', '6:18'), (None, 'b', None, None, None, None) ) diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 196ca11b..7428bb47 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -69,8 +69,8 @@ def test_number_currency(self): def test_number_currency_locale(self): rows = [ - (u'£1.7',), - (u'£200000000',), + ('£1.7',), + ('£200000000',), ('',) ] From 7945f1ad182394bbe22b0eccb718b03f45fe24f1 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:26:03 -0500 Subject: [PATCH 116/163] chore: Simplify super() calls and elif/else after returns --- agate/aggregations/count.py | 6 ++--- agate/computations/change.py | 2 +- agate/csv_py3.py | 14 ++++++------ agate/data_types/boolean.py | 14 ++++++------ agate/data_types/date.py | 5 ++-- agate/data_types/date_time.py | 8 +++---- agate/data_types/number.py | 10 ++++---- agate/data_types/text.py | 4 ++-- agate/data_types/time_delta.py | 2 +- agate/exceptions.py | 2 +- agate/mapped_sequence.py | 4 +--- agate/table/aggregate.py | 6 ++--- agate/table/order_by.py | 42 +++++++++++++++++----------------- agate/table/print_bars.py | 3 +-- agate/utils.py | 4 ++-- 15 files changed, 61 insertions(+), 65 deletions(-) diff --git a/agate/aggregations/count.py b/agate/aggregations/count.py index e7c83917..f3af86fe 100644 --- a/agate/aggregations/count.py +++ b/agate/aggregations/count.py @@ -31,7 +31,5 @@ def run(self, table): if self._column_name is not None: if self._value is not default: return table.columns[self._column_name].values().count(self._value) - else: - return len(table.columns[self._column_name].values_without_nulls()) - else: - return len(table.rows) + return len(table.columns[self._column_name].values_without_nulls()) + return len(table.rows) diff --git a/agate/computations/change.py b/agate/computations/change.py index c210e9dc..5149d229 100644 --- a/agate/computations/change.py +++ b/agate/computations/change.py @@ -29,7 +29,7 @@ def get_computed_data_type(self, table): if isinstance(before_column.data_type, (Date, DateTime, TimeDelta)): return TimeDelta() - elif isinstance(before_column.data_type, Number): + if isinstance(before_column.data_type, Number): return Number() def validate(self, table): diff --git a/agate/csv_py3.py b/agate/csv_py3.py index 9f2761f7..79b8cc8b 100644 --- a/agate/csv_py3.py +++ b/agate/csv_py3.py @@ -40,14 +40,14 @@ def __next__(self): if not self.line_numbers: return row - else: - if self.line_numbers: - if self.header and self.line_num == 1: - row.insert(0, 'line_numbers') - else: - row.insert(0, str(self.line_num - 1 if self.header else self.line_num)) - return row + if self.line_numbers: + if self.header and self.line_num == 1: + row.insert(0, 'line_numbers') + else: + row.insert(0, str(self.line_num - 1 if self.header else self.line_num)) + + return row @property def dialect(self): diff --git a/agate/data_types/boolean.py b/agate/data_types/boolean.py index 4b9b2341..8ad67cda 100644 --- a/agate/data_types/boolean.py +++ b/agate/data_types/boolean.py @@ -26,7 +26,7 @@ class Boolean(DataType): """ def __init__(self, true_values=DEFAULT_TRUE_VALUES, false_values=DEFAULT_FALSE_VALUES, null_values=DEFAULT_NULL_VALUES): - super(Boolean, self).__init__(null_values=null_values) + super().__init__(null_values=null_values) self.true_values = true_values self.false_values = false_values @@ -40,23 +40,23 @@ def cast(self, d): """ if d is None: return d - elif type(d) is bool and type(d) is not int: + if type(d) is bool and type(d) is not int: return d - elif type(d) is int or isinstance(d, Decimal): + if type(d) is int or isinstance(d, Decimal): if d == 1: return True - elif d == 0: + if d == 0: return False - elif isinstance(d, str): + if isinstance(d, str): d = d.replace(',', '').strip() d_lower = d.lower() if d_lower in self.null_values: return None - elif d_lower in self.true_values: + if d_lower in self.true_values: return True - elif d_lower in self.false_values: + if d_lower in self.false_values: return False raise CastError('Can not convert value %s to bool.' % d) diff --git a/agate/data_types/date.py b/agate/data_types/date.py index 99b526fd..639cacbe 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -23,7 +23,7 @@ class Date(DataType): for parsing formatted dates. """ def __init__(self, date_format=None, locale=None, **kwargs): - super(Date, self).__init__(**kwargs) + super().__init__(**kwargs) self.date_format = date_format self.locale = locale @@ -63,7 +63,8 @@ def cast(self, d): """ if type(d) is date or d is None: return d - elif isinstance(d, str): + + if isinstance(d, str): d = d.strip() if d.lower() in self.null_values: diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index 669cab38..a0ddaf22 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -25,7 +25,7 @@ class DateTime(DataType): for parsing formatted datetimes. """ def __init__(self, datetime_format=None, timezone=None, locale=None, **kwargs): - super(DateTime, self).__init__(**kwargs) + super().__init__(**kwargs) self.datetime_format = datetime_format self.timezone = timezone @@ -70,9 +70,9 @@ def cast(self, d): """ if isinstance(d, datetime.datetime) or d is None: return d - elif isinstance(d, datetime.date): + if isinstance(d, datetime.date): return datetime.datetime.combine(d, datetime.time(0, 0, 0)) - elif isinstance(d, str): + if isinstance(d, str): d = d.strip() if d.lower() in self.null_values: @@ -109,7 +109,7 @@ def cast(self, d): if matched_text == d and ctx.hasDate and ctx.hasTime: return value - elif matched_text == d and ctx.hasDate and not ctx.hasTime: + if matched_text == d and ctx.hasDate and not ctx.hasTime: return datetime.datetime.combine(value.date(), datetime.time.min) try: diff --git a/agate/data_types/number.py b/agate/data_types/number.py index 2d5a7786..ab8ed8fa 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -35,7 +35,7 @@ class Number(DataType): """ def __init__(self, locale='en_US', group_symbol=None, decimal_symbol=None, currency_symbols=DEFAULT_CURRENCY_SYMBOLS, **kwargs): - super(Number, self).__init__(**kwargs) + super().__init__(**kwargs) self.locale = Locale.parse(locale) @@ -63,13 +63,13 @@ def cast(self, d): if t is int: return Decimal(d) - elif t is float: + if t is float: return Decimal(repr(d)) - elif d is False: + if d is False: return Decimal(0) - elif d is True: + if d is True: return Decimal(1) - elif not isinstance(d, str): + if not isinstance(d, str): raise CastError('Can not parse value "%s" as Decimal.' % d) d = d.strip() diff --git a/agate/data_types/text.py b/agate/data_types/text.py index ed31bef5..be5f5711 100644 --- a/agate/data_types/text.py +++ b/agate/data_types/text.py @@ -13,7 +13,7 @@ class Text(DataType): converted to `None`. Disable to retain them as strings. """ def __init__(self, cast_nulls=True, **kwargs): - super(Text, self).__init__(**kwargs) + super().__init__(**kwargs) self.cast_nulls = cast_nulls @@ -28,7 +28,7 @@ def cast(self, d): """ if d is None: return d - elif isinstance(d, str): + if isinstance(d, str): if self.cast_nulls and d.strip().lower() in self.null_values: return None diff --git a/agate/data_types/time_delta.py b/agate/data_types/time_delta.py index e6e49718..49ccbeed 100644 --- a/agate/data_types/time_delta.py +++ b/agate/data_types/time_delta.py @@ -23,7 +23,7 @@ def cast(self, d): """ if isinstance(d, datetime.timedelta) or d is None: return d - elif isinstance(d, str): + if isinstance(d, str): d = d.strip() if d.lower() in self.null_values: diff --git a/agate/exceptions.py b/agate/exceptions.py index 0673c7c4..faaaee70 100644 --- a/agate/exceptions.py +++ b/agate/exceptions.py @@ -35,7 +35,7 @@ class FieldSizeLimitError(Exception): # pragma: no cover This length may be the default or one set by the user. """ def __init__(self, limit, line_number): - super(FieldSizeLimitError, self).__init__( + super().__init__( 'CSV contains a field longer than the maximum length of %i characters on line %i. Try raising the maximum ' 'with the field_size_limit parameter, or try setting quoting=csv.QUOTE_NONE.' % (limit, line_number) ) diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index 2ca22995..1df5cf77 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -83,13 +83,11 @@ def __getitem__(self, key): if isinstance(key, slice): indices = range(*key.indices(len(self))) values = self.values() - return tuple(values[i] for i in indices) # Note: can't use isinstance because bool is a subclass of int elif type(key) is int: return self.values()[key] - else: - return self.dict()[key] + return self.dict()[key] def __setitem__(self, key, value): """ diff --git a/agate/table/aggregate.py b/agate/table/aggregate.py index f6c04377..66878513 100644 --- a/agate/table/aggregate.py +++ b/agate/table/aggregate.py @@ -29,7 +29,7 @@ def aggregate(self, aggregations): results[name] = agg.run(self) return results - else: - aggregations.validate(self) - return aggregations.run(self) + aggregations.validate(self) + + return aggregations.run(self) diff --git a/agate/table/order_by.py b/agate/table/order_by.py index 80f93ce3..8ceff4df 100644 --- a/agate/table/order_by.py +++ b/agate/table/order_by.py @@ -19,32 +19,32 @@ def order_by(self, key, reverse=False): """ if len(self._rows) == 0: return self._fork(self._rows) - else: - key_is_row_function = hasattr(key, '__call__') - key_is_sequence = utils.issequence(key) - def sort_key(data): - row = data[1] + key_is_row_function = hasattr(key, '__call__') + key_is_sequence = utils.issequence(key) - if key_is_row_function: - k = key(row) - elif key_is_sequence: - k = tuple(utils.NullOrder() if row[n] is None else row[n] for n in key) - else: - k = row[key] + def sort_key(data): + row = data[1] - if k is None: - return utils.NullOrder() + if key_is_row_function: + k = key(row) + elif key_is_sequence: + k = tuple(utils.NullOrder() if row[n] is None else row[n] for n in key) + else: + k = row[key] - return k + if k is None: + return utils.NullOrder() - results = sorted(enumerate(self._rows), key=sort_key, reverse=reverse) + return k - indices, rows = zip(*results) + results = sorted(enumerate(self._rows), key=sort_key, reverse=reverse) - if self._row_names is not None: - row_names = [self._row_names[i] for i in indices] - else: - row_names = None + indices, rows = zip(*results) + + if self._row_names is not None: + row_names = [self._row_names[i] for i in indices] + else: + row_names = None - return self._fork(rows, row_names=row_names) + return self._fork(rows, row_names=row_names) diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index 2dba87de..c060fbca 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -126,8 +126,7 @@ def print_bars(self, label_column_name='group', value_column_name='Count', domai def project(value): if value >= 0: return plot_negative_width + int((plot_positive_width * (value / x_max)).to_integral_value()) - else: - return plot_negative_width - int((plot_negative_width * (value / x_min)).to_integral_value()) + return plot_negative_width - int((plot_negative_width * (value / x_min)).to_integral_value()) # Calculate ticks ticks = OrderedDict() diff --git a/agate/utils.py b/agate/utils.py index a51fe5f8..0fd22061 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -312,5 +312,5 @@ def slugify(values, ensure_unique=False, **kwargs): if ensure_unique: new_values = tuple(pslugify(value, **slug_args) for value in values) return deduplicate(new_values, separator=slug_args['separator']) - else: - return tuple(pslugify(value, **slug_args) for value in values) + + return tuple(pslugify(value, **slug_args) for value in values) From a82b43cfd772e798afb2ffbd288d07e6619ff90f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:36:16 -0500 Subject: [PATCH 117/163] chore: Fix misconversion from c83d726d535dcec08b0c5b1301b8671068c25b12 --- agate/csv_py2.py | 273 ---------------------------------------- agate/table/from_csv.py | 2 +- 2 files changed, 1 insertion(+), 274 deletions(-) delete mode 100644 agate/csv_py2.py diff --git a/agate/csv_py2.py b/agate/csv_py2.py deleted file mode 100644 index 568d2603..00000000 --- a/agate/csv_py2.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python - -""" -This module contains the Python 2 replacement for :mod:`csv`. -""" - -import codecs -import csv -import warnings -from io import StringIO - -from agate.exceptions import FieldSizeLimitError - -EIGHT_BIT_ENCODINGS = [ - 'utf-8', 'u8', 'utf', 'utf8', - 'latin-1', 'iso-8859-1', 'iso8859-1', '8859', 'cp819', 'latin', 'latin1', 'l1' -] - -POSSIBLE_DELIMITERS = [',', '\t', ';', ' ', ':', '|'] - - -class UTF8Recoder: - """ - Iterator that reads an encoded stream and reencodes the input to UTF-8. - """ - def __init__(self, f, encoding): - self.reader = codecs.getreader(encoding)(f) - - def __iter__(self): - return self - - def __next__(self): - return next(self.reader).encode('utf-8') - - -class UnicodeReader: - """ - A CSV reader which will read rows from a file in a given encoding. - """ - def __init__(self, f, encoding='utf-8', field_size_limit=None, line_numbers=False, header=True, **kwargs): - self.line_numbers = line_numbers - self.header = header - - f = UTF8Recoder(f, encoding) - - self.reader = csv.reader(f, **kwargs) - - if field_size_limit: - csv.field_size_limit(field_size_limit) - - def next(self): - try: - row = next(self.reader) - except csv.Error as e: - # Terrible way to test for this exception, but there is no subclass - if 'field larger than field limit' in str(e): - raise FieldSizeLimitError(csv.field_size_limit(), self.line_num) - else: - raise e - - if self.line_numbers: - if self.header and self.line_num == 1: - row.insert(0, 'line_numbers') - else: - row.insert(0, str(self.line_num - 1 if self.header else self.line_num)) - - return [str(s, 'utf-8') for s in row] - - def __iter__(self): - return self - - @property - def dialect(self): - return self.reader.dialect - - @property - def line_num(self): - return self.reader.line_num - - -class UnicodeWriter: - """ - A CSV writer which will write rows to a file in the specified encoding. - - NB: Optimized so that eight-bit encodings skip re-encoding. See: - https://github.com/wireservice/csvkit/issues/175 - """ - def __init__(self, f, encoding='utf-8', **kwargs): - self.encoding = encoding - self._eight_bit = (self.encoding.lower().replace('_', '-') in EIGHT_BIT_ENCODINGS) - - if self._eight_bit: - self.writer = csv.writer(f, **kwargs) - else: - # Redirect output to a queue for reencoding - self.queue = StringIO() - self.writer = csv.writer(self.queue, **kwargs) - self.stream = f - self.encoder = codecs.getincrementalencoder(encoding)() - - def writerow(self, row): - if self._eight_bit: - self.writer.writerow([str(s if s is not None else '').encode(self.encoding) for s in row]) - else: - self.writer.writerow([str(s if s is not None else '').encode('utf-8') for s in row]) - # Fetch UTF-8 output from the queue... - data = self.queue.getvalue() - data = data.decode('utf-8') - # ...and reencode it into the target encoding - data = self.encoder.encode(data) - # write to the file - self.stream.write(data) - # empty the queue - self.queue.truncate(0) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - - -class UnicodeDictReader(csv.DictReader): - """ - Defer almost all implementation to :class:`csv.DictReader`, but wraps our - unicode reader instead of :func:`csv.reader`. - """ - def __init__(self, f, fieldnames=None, restkey=None, restval=None, *args, **kwargs): - reader = UnicodeReader(f, *args, **kwargs) - - if 'encoding' in kwargs: - kwargs.pop('encoding') - - csv.DictReader.__init__(self, f, fieldnames, restkey, restval, *args, **kwargs) - - self.reader = reader - - -class UnicodeDictWriter(csv.DictWriter): - """ - Defer almost all implementation to :class:`csv.DictWriter`, but wraps our - unicode writer instead of :func:`csv.writer`. - """ - def __init__(self, f, fieldnames, restval='', extrasaction='raise', *args, **kwds): - self.fieldnames = fieldnames - self.restval = restval - - if extrasaction.lower() not in ('raise', 'ignore'): - raise ValueError('extrasaction (%s) must be "raise" or "ignore"' % extrasaction) - - self.extrasaction = extrasaction - - self.writer = UnicodeWriter(f, *args, **kwds) - - -class Reader(UnicodeReader): - """ - A unicode-aware CSV reader. - """ - pass - - -class Writer(UnicodeWriter): - """ - A unicode-aware CSV writer. - """ - def __init__(self, f, encoding='utf-8', line_numbers=False, **kwargs): - self.row_count = 0 - self.line_numbers = line_numbers - - if 'lineterminator' not in kwargs: - kwargs['lineterminator'] = '\n' - - UnicodeWriter.__init__(self, f, encoding, **kwargs) - - def _append_line_number(self, row): - if self.row_count == 0: - row.insert(0, 'line_number') - else: - row.insert(0, self.row_count) - - self.row_count += 1 - - def writerow(self, row): - if self.line_numbers: - row = list(row) - self._append_line_number(row) - - # Convert embedded Mac line endings to unix style line endings so they get quoted - row = [i.replace('\r', '\n') if isinstance(i, str) else i for i in row] - - UnicodeWriter.writerow(self, row) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - - -class DictReader(UnicodeDictReader): - """ - A unicode-aware CSV DictReader. - """ - pass - - -class DictWriter(UnicodeDictWriter): - """ - A unicode-aware CSV DictWriter. - """ - def __init__(self, f, fieldnames, encoding='utf-8', line_numbers=False, **kwargs): - self.row_count = 0 - self.line_numbers = line_numbers - - if 'lineterminator' not in kwargs: - kwargs['lineterminator'] = '\n' - - UnicodeDictWriter.__init__(self, f, fieldnames, encoding=encoding, **kwargs) - - def _append_line_number(self, row): - if self.row_count == 0: - row['line_number'] = 0 - else: - row['line_number'] = self.row_count - - self.row_count += 1 - - def writerow(self, row): - if self.line_numbers: - row = list(row) - self._append_line_number(row) - - # Convert embedded Mac line endings to unix style line endings so they get quoted - row = dict([ - (k, v.replace('\r', '\n')) if isinstance(v, basestring) else (k, v) for k, v in row.items() # noqa: F821 - ]) - - UnicodeDictWriter.writerow(self, row) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - - -class Sniffer: - """ - A functional wrapper of ``csv.Sniffer()``. - """ - def sniff(self, sample): - """ - A functional version of ``csv.Sniffer().sniff``, that extends the - list of possible delimiters to include some seen in the wild. - """ - try: - dialect = csv.Sniffer().sniff(sample, POSSIBLE_DELIMITERS) - except csv.Error as e: - warnings.warn('Error sniffing CSV dialect: %s' % e, RuntimeWarning, stacklevel=2) - dialect = None - - return dialect - - -def reader(*args, **kwargs): - """ - A replacement for Python's :func:`csv.reader` that uses - :class:`.csv_py2.Reader`. - """ - return Reader(*args, **kwargs) - - -def writer(*args, **kwargs): - """ - A replacement for Python's :func:`csv.writer` that uses - :class:`.csv_py2.Writer`. - """ - return Writer(*args, **kwargs) diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index e6ae7288..368e7e85 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -69,7 +69,7 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk elif sniff_limit > 0: kwargs['dialect'] = csv.Sniffer().sniff(contents.getvalue()[:sniff_limit]) - reader = csv.reader(contents, header=header, encoding=encoding, **kwargs) + reader = csv.reader(contents, header=header, **kwargs) if header: if column_names is None: From 4d5267e8e58e21f50977822923d14cf364aa07d8 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:40:20 -0500 Subject: [PATCH 118/163] ci: No duplicate builds --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdd2d117..472e7576 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI on: [push, pull_request] jobs: build: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ${{ matrix.os }} strategy: matrix: From a1e2aaf2cd851d5020e692246fe5f07586f878ea Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Jan 2023 13:12:45 -0500 Subject: [PATCH 119/163] docs: Update changelog --- CHANGELOG.rst | 5 +++++ docs/conf.py | 4 ++-- setup.py | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 88100649..881ad0b5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +1.7.0 - Jan 3, 2023 +------------------- + +- Drop support for Python 2.7 (no longer works) and 3.6 (no longer tested). + 1.6.3 - July 15, 2021 --------------------- diff --git a/docs/conf.py b/docs/conf.py index 9b9bdd0b..8b6cad85 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,7 +17,7 @@ autodoc_default_flags = ['members', 'show-inheritance'] intersphinx_mapping = { - 'python': ('http://docs.python.org/3.5', None), + 'python': ('http://docs.python.org/3.11', None), 'leather': ('http://leather.readthedocs.io/en/latest/', None) } @@ -28,7 +28,7 @@ # Metadata project = 'agate' copyright = '2017, Christopher Groskopf' -version = '1.6.3' +version = '1.7.0' release = version exclude_patterns = ['_build'] diff --git a/setup.py b/setup.py index 7f43bcfd..fe694ef4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='agate', - version='1.6.3', + version='1.7.0', description='A data analysis library that is optimized for humans instead of machines.', long_description=long_description, long_description_content_type='text/x-rst', @@ -25,11 +25,11 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Scientific/Engineering :: Information Analysis', From 32cce432c5da12cadb947e4b52321a6bd6303c79 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Jan 2023 13:15:46 -0500 Subject: [PATCH 120/163] docs: Update badges --- README.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 54bc5795..9240443d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,11 @@ -.. image:: https://travis-ci.org/wireservice/agate.png - :target: https://travis-ci.org/wireservice/agate +.. image:: https://github.com/wireservice/agate/workflows/CI/badge.svg + :target: https://github.com/wireservice/agate/actions :alt: Build status +.. image:: https://img.shields.io/pypi/dm/agate.svg + :target: https://pypi.python.org/pypi/agate + :alt: PyPI downloads + .. image:: https://img.shields.io/pypi/v/agate.svg :target: https://pypi.python.org/pypi/agate :alt: Version From cd637923b5e3aa62aebfaba961303bbd66f47289 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Jan 2023 13:25:00 -0500 Subject: [PATCH 121/163] docs: Update changelog --- CHANGELOG.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 881ad0b5..c3a104b0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,10 @@ 1.7.0 - Jan 3, 2023 ------------------- -- Drop support for Python 2.7 (no longer works) and 3.6 (no longer tested). +* Add Python 3.11 support. +* Add Python 3.10 support. +* Drop Python 3.6 support (end-of-life was December 23, 2021). +* Drop Python 2.7 support (end-of-life was January 1, 2020). 1.6.3 - July 15, 2021 --------------------- @@ -22,6 +25,8 @@ * fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: Ensure files are closed when errors occur. (#734) * build: Make PyICU an optional dependency. +* Drop Python 3.5 support (end-of-life was September 13, 2020). +* Drop Python 3.4 support (end-of-life was March 18, 2019). 1.6.2 - March 10, 2021 ---------------------- From 52198daae198389649a44deb0ec2d1d41f6720c1 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:11:59 -0500 Subject: [PATCH 122/163] build: Allow parsedatetime 2.6 --- CHANGELOG.rst | 5 +++++ docs/conf.py | 2 +- setup.py | 5 ++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c3a104b0..0d7ddb52 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +1.7.1 - Jan 4, 2023 +------------------- + +* Allow parsedatetime 2.6. + 1.7.0 - Jan 3, 2023 ------------------- diff --git a/docs/conf.py b/docs/conf.py index 8b6cad85..60ba2ddd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ # Metadata project = 'agate' copyright = '2017, Christopher Groskopf' -version = '1.7.0' +version = '1.7.1' release = version exclude_patterns = ['_build'] diff --git a/setup.py b/setup.py index fe694ef4..f6541491 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='agate', - version='1.7.0', + version='1.7.1', description='A data analysis library that is optimized for humans instead of machines.', long_description=long_description, long_description_content_type='text/x-rst', @@ -41,8 +41,7 @@ 'isodate>=0.5.4', 'leather>=0.3.2', # KeyError: 's' https://github.com/bear/parsedatetime/pull/233 https://github.com/wireservice/agate/issues/743 - # AttributeError: 'module' object has no attribute 'Locale' https://github.com/bear/parsedatetime/pull/247 - 'parsedatetime>=2.1,!=2.5,!=2.6', + 'parsedatetime>=2.1,!=2.5', 'python-slugify>=1.2.1', 'pytimeparse>=1.1.5', ], From a5dc7bbaa0292c1cb8741b559d5dab618f5bd2f0 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 12 Jan 2023 01:10:52 -0500 Subject: [PATCH 123/163] chore: Run pyupgrade --py36-plus **/*.py --- agate/aggregations/first.py | 2 +- agate/config.py | 1 - agate/data_types/number.py | 1 - agate/mapped_sequence.py | 2 +- agate/table/__init__.py | 2 +- agate/table/bins.py | 2 +- agate/table/from_csv.py | 3 +-- agate/table/from_fixed.py | 6 ++---- agate/table/from_json.py | 5 ++--- agate/table/print_bars.py | 5 ++--- agate/table/print_table.py | 10 +++++----- agate/tableset/from_csv.py | 2 +- agate/tableset/from_json.py | 4 ++-- agate/utils.py | 1 - agate/warns.py | 4 ++-- benchmarks/test_joins.py | 1 - docs/conf.py | 1 - tests/test_aggregations.py | 1 - tests/test_columns.py | 1 - tests/test_data_types.py | 1 - tests/test_from_json.py | 1 - tests/test_py3.py | 9 ++++----- tests/test_table/__init__.py | 1 - tests/test_table/test_aggregate.py | 1 - tests/test_table/test_bins.py | 1 - tests/test_table/test_charting.py | 1 - tests/test_table/test_compute.py | 1 - tests/test_table/test_denormalize.py | 1 - tests/test_table/test_from_csv.py | 4 +--- tests/test_table/test_group_by.py | 1 - tests/test_table/test_homogenize.py | 1 - tests/test_table/test_join.py | 1 - tests/test_table/test_merge.py | 1 - tests/test_table/test_normalize.py | 1 - tests/test_table/test_order_py.py | 1 - tests/test_table/test_pivot.py | 1 - tests/test_table/test_print_bars.py | 1 - tests/test_table/test_print_html.py | 1 - tests/test_table/test_print_structure.py | 1 - tests/test_table/test_print_table.py | 1 - tests/test_table/test_rename.py | 1 - tests/test_table/test_to_csv.py | 1 - tests/test_table/test_to_json.py | 1 - tests/test_tableset/test_charting.py | 1 - tests/test_type_tester.py | 1 - 45 files changed, 26 insertions(+), 65 deletions(-) diff --git a/agate/aggregations/first.py b/agate/aggregations/first.py index 287bfa13..32123b41 100644 --- a/agate/aggregations/first.py +++ b/agate/aggregations/first.py @@ -38,4 +38,4 @@ def run(self, table): if self._test is None: return data[0] - return next((d for d in data if self._test(d))) + return next(d for d in data if self._test(d)) diff --git a/agate/config.py b/agate/config.py index c28ef6c5..c99716b7 100644 --- a/agate/config.py +++ b/agate/config.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- """ This module contains the global configuration for agate. Users should use diff --git a/agate/data_types/number.py b/agate/data_types/number.py index ab8ed8fa..b6b73b2d 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import warnings from decimal import Decimal, InvalidOperation diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index 1df5cf77..7878d621 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -65,7 +65,7 @@ def __unicode__(self): if len(self) > 5: sample = '%s, ...' % sample - return '' % (type(self).__name__, sample) + return ''.format(type(self).__name__, sample) def __str__(self): """ diff --git a/agate/table/__init__.py b/agate/table/__init__.py index 918f6478..8cb3b121 100644 --- a/agate/table/__init__.py +++ b/agate/table/__init__.py @@ -130,7 +130,7 @@ def __init__(self, rows, column_names=None, column_types=None, row_names=None, _ try: row_values.append(cast_funcs[j](d)) except CastError as e: - raise CastError(str(e) + ' Error at row %s column %s.' % (i, self._column_names[j])) + raise CastError(str(e) + ' Error at row {} column {}.'.format(i, self._column_names[j])) new_rows.append(Row(row_values, self._column_names)) else: diff --git a/agate/table/bins.py b/agate/table/bins.py index e05e935d..1f35c3cd 100644 --- a/agate/table/bins.py +++ b/agate/table/bins.py @@ -63,7 +63,7 @@ def name_bin(i, j, first_exclusive=True, last_exclusive=False): exclusive = format_decimal(j, format=break_formatter) output = '[' if first_exclusive else '(' - output += '%s - %s' % (inclusive, exclusive) + output += '{} - {}'.format(inclusive, exclusive) output += ']' if last_exclusive else ')' return output diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index 368e7e85..b8a6297a 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import io import itertools from io import StringIO @@ -51,7 +50,7 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk if hasattr(path, 'read'): f = path else: - f = io.open(path, encoding=encoding) + f = open(path, encoding=encoding) close = True diff --git a/agate/table/from_fixed.py b/agate/table/from_fixed.py index c0fa4da4..0e2653b6 100644 --- a/agate/table/from_fixed.py +++ b/agate/table/from_fixed.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -import io - from agate import fixed, utils @@ -42,13 +40,13 @@ def from_fixed(cls, path, schema_path, column_names=utils.default, column_types= try: if not hasattr(path, 'read'): - f = io.open(path, encoding=encoding) + f = open(path, encoding=encoding) close_f = True else: f = path if not hasattr(schema_path, 'read'): - schema_f = io.open(schema_path, encoding=schema_encoding) + schema_f = open(schema_path, encoding=schema_encoding) close_schema_f = True else: schema_f = path diff --git a/agate/table/from_json.py b/agate/table/from_json.py index 2b673f1e..eacb8618 100644 --- a/agate/table/from_json.py +++ b/agate/table/from_json.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import io import json from collections import OrderedDict from decimal import Decimal @@ -51,7 +50,7 @@ def from_json(cls, path, row_names=None, key=None, newline=False, column_types=N for line in path: js.append(json.loads(line, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs)) else: - f = io.open(path, encoding=encoding) + f = open(path, encoding=encoding) close = True for line in f: @@ -60,7 +59,7 @@ def from_json(cls, path, row_names=None, key=None, newline=False, column_types=N if hasattr(path, 'read'): js = json.load(path, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs) else: - f = io.open(path, encoding=encoding) + f = open(path, encoding=encoding) close = True js = json.load(f, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs) diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index c060fbca..23f89348 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- # pylint: disable=W0212 import sys @@ -176,7 +175,7 @@ def write(line): output.write(line + '\n') # Chart top - top_line = '%s %s' % (y_label.ljust(max_label_width), x_label.rjust(max_value_width)) + top_line = '{} {}'.format(y_label.ljust(max_label_width), x_label.rjust(max_value_width)) write(top_line) # Bars @@ -211,7 +210,7 @@ def write(line): bar = bar.ljust(plot_width) - write('%s %s %s' % (label_text, value_text, bar)) + write('{} {} {}'.format(label_text, value_text, bar)) # Axis & ticks axis = horizontal_line * plot_width diff --git a/agate/table/print_table.py b/agate/table/print_table.py index eece96ab..0ad5c5af 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -137,12 +137,12 @@ def write_row(formatted_row): text = v_line.join(row_output) - write('%s%s%s' % (v_line, text, v_line)) + write('{}{}{}'.format(v_line, text, v_line)) - divider = '%(v_line)s %(columns)s %(v_line)s' % { - 'v_line': v_line, - 'columns': ' | '.join(h_line * w for w in widths) - } + divider = '{v_line} {columns} {v_line}'.format( + v_line=v_line, + columns=' | '.join(h_line * w for w in widths) + ) # Headers write_row(column_names) diff --git a/agate/tableset/from_csv.py b/agate/tableset/from_csv.py index 64b7e2e8..cde22bdb 100644 --- a/agate/tableset/from_csv.py +++ b/agate/tableset/from_csv.py @@ -29,7 +29,7 @@ def from_csv(cls, dir_path, column_names=None, column_types=None, row_names=None from agate.tableset import TableSet if not os.path.isdir(dir_path): - raise IOError('Specified path doesn\'t exist or isn\'t a directory.') + raise OSError('Specified path doesn\'t exist or isn\'t a directory.') tables = OrderedDict() diff --git a/agate/tableset/from_json.py b/agate/tableset/from_json.py index e83205ef..bb79074f 100644 --- a/agate/tableset/from_json.py +++ b/agate/tableset/from_json.py @@ -30,7 +30,7 @@ def from_json(cls, path, column_names=None, column_types=None, keys=None, **kwar from agate.tableset import TableSet if isinstance(path, str) and not os.path.isdir(path) and not os.path.isfile(path): - raise IOError('Specified path doesn\'t exist.') + raise OSError('Specified path doesn\'t exist.') tables = OrderedDict() @@ -52,7 +52,7 @@ def from_json(cls, path, column_names=None, column_types=None, keys=None, **kwar if hasattr(path, 'read'): js = json.load(path, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs) else: - with open(path, 'r') as f: + with open(path) as f: js = json.load(f, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs) for key, value in js.items(): diff --git a/agate/utils.py b/agate/utils.py index 0fd22061..e95d256f 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- """ This module contains a collection of utility classes and functions used in diff --git a/agate/warns.py b/agate/warns.py index 1106f7f9..53cd408d 100644 --- a/agate/warns.py +++ b/agate/warns.py @@ -13,7 +13,7 @@ class NullCalculationWarning(RuntimeWarning): # pragma: no cover def warn_null_calculation(operation, column): - warnings.warn('Column "%s" contains nulls. These will be excluded from %s calculation.' % ( + warnings.warn('Column "{}" contains nulls. These will be excluded from {} calculation.'.format( column.name, operation.__class__.__name__ ), NullCalculationWarning, stacklevel=2) @@ -28,7 +28,7 @@ class DuplicateColumnWarning(RuntimeWarning): # pragma: no cover def warn_duplicate_column(column_name, column_rename): - warnings.warn('Column name "%s" already exists in Table. Column will be renamed to "%s".' % ( + warnings.warn('Column name "{}" already exists in Table. Column will be renamed to "{}".'.format( column_name, column_rename ), DuplicateColumnWarning, stacklevel=2) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 211df19d..7d27d768 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import unittest from random import shuffle diff --git a/docs/conf.py b/docs/conf.py index 60ba2ddd..5d264822 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import os import sys diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index 239b7ffb..e1645ed7 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import datetime import sys diff --git a/tests/test_columns.py b/tests/test_columns.py index 2b04e311..cec1b8f2 100644 --- a/tests/test_columns.py +++ b/tests/test_columns.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import pickle import unittest diff --git a/tests/test_data_types.py b/tests/test_data_types.py index 6e76b749..ae4b7de2 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import datetime import pickle diff --git a/tests/test_from_json.py b/tests/test_from_json.py index d20aa30a..a99a2173 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table diff --git a/tests/test_py3.py b/tests/test_py3.py index 24b852d0..6f210077 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import csv import os @@ -74,7 +73,7 @@ def tearDown(self): def test_field_size_limit(self): # Testing field_size_limit for failure. Creating data using str * int. - with open('.test.csv', 'r', encoding='utf-8') as f: + with open('.test.csv', encoding='utf-8') as f: c = csv_py3.Reader(f, field_size_limit=9) try: c.__next__() @@ -84,7 +83,7 @@ def test_field_size_limit(self): raise AssertionError('Expected FieldSizeLimitError') # Now testing higher field_size_limit. - with open('.test.csv', 'r', encoding='utf-8') as f: + with open('.test.csv', encoding='utf-8') as f: c = csv_py3.Reader(f, field_size_limit=11) self.assertEqual(['a' * 10], c.__next__()) @@ -247,5 +246,5 @@ def test_sniffer(self): actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ - self.assertEqual(direct, expected, '%r != %r' % (direct, expected)) - self.assertEqual(actual, expected, '%r != %r' % (actual, expected)) + self.assertEqual(direct, expected, '{!r} != {!r}'.format(direct, expected)) + self.assertEqual(actual, expected, '{!r} != {!r}'.format(actual, expected)) diff --git a/tests/test_table/__init__.py b/tests/test_table/__init__.py index 882179d8..4b582dad 100644 --- a/tests/test_table/__init__.py +++ b/tests/test_table/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import warnings from decimal import Decimal diff --git a/tests/test_table/test_aggregate.py b/tests/test_table/test_aggregate.py index 4ded61f5..2052b363 100644 --- a/tests/test_table/test_aggregate.py +++ b/tests/test_table/test_aggregate.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table from agate.aggregations import Count, Sum diff --git a/tests/test_table/test_bins.py b/tests/test_table/test_bins.py index c60f3e81..f3395b23 100644 --- a/tests/test_table/test_bins.py +++ b/tests/test_table/test_bins.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from decimal import Decimal diff --git a/tests/test_table/test_charting.py b/tests/test_table/test_charting.py index d9ea7d08..2663a1fa 100644 --- a/tests/test_table/test_charting.py +++ b/tests/test_table/test_charting.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import leather diff --git a/tests/test_table/test_compute.py b/tests/test_table/test_compute.py index a18d7e48..42e4b409 100644 --- a/tests/test_table/test_compute.py +++ b/tests/test_table/test_compute.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table diff --git a/tests/test_table/test_denormalize.py b/tests/test_table/test_denormalize.py index d85f878c..a27ab0e7 100644 --- a/tests/test_table/test_denormalize.py +++ b/tests/test_table/test_denormalize.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index 367e1634..2384849a 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- -import io import warnings from agate import Table @@ -56,7 +54,7 @@ def test_from_csv_cr(self): def test_from_csv_file_like_object(self): table1 = Table(self.rows, self.column_names, self.column_types) - f = io.open('examples/test.csv', encoding='utf-8') + f = open('examples/test.csv', encoding='utf-8') table2 = Table.from_csv(f) f.close() diff --git a/tests/test_table/test_group_by.py b/tests/test_table/test_group_by.py index 05b9f12c..3b0e278b 100644 --- a/tests/test_table/test_group_by.py +++ b/tests/test_table/test_group_by.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from decimal import Decimal diff --git a/tests/test_table/test_homogenize.py b/tests/test_table/test_homogenize.py index 976f74db..1183aa06 100644 --- a/tests/test_table/test_homogenize.py +++ b/tests/test_table/test_homogenize.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_table/test_join.py b/tests/test_table/test_join.py index f7655c61..763ca503 100644 --- a/tests/test_table/test_join.py +++ b/tests/test_table/test_join.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_table/test_merge.py b/tests/test_table/test_merge.py index f2f84812..b9d823f4 100644 --- a/tests/test_table/test_merge.py +++ b/tests/test_table/test_merge.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_table/test_normalize.py b/tests/test_table/test_normalize.py index 9d131843..4ba0252d 100644 --- a/tests/test_table/test_normalize.py +++ b/tests/test_table/test_normalize.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_table/test_order_py.py b/tests/test_table/test_order_py.py index 3f29fbe0..6692a025 100644 --- a/tests/test_table/test_order_py.py +++ b/tests/test_table/test_order_py.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from agate import Table from agate.data_types import Number, Text diff --git a/tests/test_table/test_pivot.py b/tests/test_table/test_pivot.py index 48de7ef8..dc67f926 100644 --- a/tests/test_table/test_pivot.py +++ b/tests/test_table/test_pivot.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import sys from decimal import Decimal diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index 5c473dd6..d80ad590 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from io import StringIO diff --git a/tests/test_table/test_print_html.py b/tests/test_table/test_print_html.py index d1b67057..3e9cab47 100644 --- a/tests/test_table/test_print_html.py +++ b/tests/test_table/test_print_html.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import warnings from html.parser import HTMLParser diff --git a/tests/test_table/test_print_structure.py b/tests/test_table/test_print_structure.py index 7d742843..4966ddcb 100644 --- a/tests/test_table/test_print_structure.py +++ b/tests/test_table/test_print_structure.py @@ -1,5 +1,4 @@ # !/usr/bin/env python -# -*- coding: utf8 -*- from io import StringIO diff --git a/tests/test_table/test_print_table.py b/tests/test_table/test_print_table.py index 187b41c0..5061e5a3 100644 --- a/tests/test_table/test_print_table.py +++ b/tests/test_table/test_print_table.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from io import StringIO diff --git a/tests/test_table/test_rename.py b/tests/test_table/test_rename.py index c5740d80..2ddc6f84 100644 --- a/tests/test_table/test_rename.py +++ b/tests/test_table/test_rename.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import warnings diff --git a/tests/test_table/test_to_csv.py b/tests/test_table/test_to_csv.py index 525d336c..11ace106 100644 --- a/tests/test_table/test_to_csv.py +++ b/tests/test_table/test_to_csv.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import os import sys diff --git a/tests/test_table/test_to_json.py b/tests/test_table/test_to_json.py index 21af2515..023d3f4f 100644 --- a/tests/test_table/test_to_json.py +++ b/tests/test_table/test_to_json.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import json import os diff --git a/tests/test_tableset/test_charting.py b/tests/test_tableset/test_charting.py index e90bf227..90758e34 100644 --- a/tests/test_tableset/test_charting.py +++ b/tests/test_tableset/test_charting.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- from collections import OrderedDict diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 7428bb47..4dde9b06 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- import unittest From a1e23231c7de875f42b7b41a7ab528782003c5ba Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:07:40 -0400 Subject: [PATCH 124/163] test: Restore test_sniffer, closes #757 --- tests/test_py3.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_py3.py b/tests/test_py3.py index 6f210077..f3a561a5 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -2,7 +2,6 @@ import csv import os -import platform import unittest from io import StringIO @@ -233,15 +232,10 @@ def test_writerows(self): self.assertEqual(result, 'line_number,a,b,c\n1,1,2,☃\n') -@unittest.skipIf(platform.system() == 'Linux', - "Test inexplicably fails intermittently on Linux.") class TestSniffer(unittest.TestCase): def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() - # Failure observed on Python 3.6 and 3.8. The left-hand side in these cases is an empty dictionary. - # 3.6 https://github.com/wireservice/agate/runs/3078380985 - # 3.8 https://github.com/wireservice/agate/runs/3078633299 direct = csv.Sniffer().sniff(contents, csv_py3.POSSIBLE_DELIMITERS).__dict__ actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ From 595718a766008eb2ed0081c550d1abd598b8b7bd Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:45:18 -0400 Subject: [PATCH 125/163] chore: Add .readthedocs.yaml --- .readthedocs.yaml | 11 +++++++++++ MANIFEST.in | 2 ++ docs/requirements.txt | 3 +++ 3 files changed, 16 insertions(+) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..8ea94712 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,11 @@ +version: 2 +build: + os: ubuntu-20.04 + tools: + python: "3.9" +python: + install: + - path: . + - requirements: docs/requirements.txt +sphinx: + fail_on_warning: true diff --git a/MANIFEST.in b/MANIFEST.in index bddc21f1..72f9db54 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,10 +6,12 @@ recursive-include benchmarks *.py recursive-include docs *.py recursive-include docs *.rst recursive-include docs *.svg +recursive-include docs *.txt recursive-include docs Makefile recursive-include examples *.csv recursive-include examples *.json recursive-include examples testfixed recursive-include tests *.py exclude .pre-commit-config.yaml +exclude .readthedocs.yaml global-exclude *.pyc diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..a669e4fa --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +furo +sphinx>2 +docutils>=0.18 From 0aa330053652221bdfd085558b46578c14537d69 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:54:55 -0400 Subject: [PATCH 126/163] docs: Update theme --- docs/conf.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5d264822..01a4620a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,14 +34,7 @@ pygments_style = 'sphinx' # HTMl theming -html_theme = 'default' - -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_theme = 'furo' html_static_path = ['_static'] htmlhelp_basename = 'agatedoc' From 8ada219fa8f8c9bab636f5f2f6250ae2f621ed32 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:58:40 -0400 Subject: [PATCH 127/163] docs: Update intersphinx_mapping --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 01a4620a..56adadfa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ autodoc_default_flags = ['members', 'show-inheritance'] intersphinx_mapping = { - 'python': ('http://docs.python.org/3.11', None), + 'python': ('https://docs.python.org/3.11', None), 'leather': ('http://leather.readthedocs.io/en/latest/', None) } From b691dbfa56a42807f189e77227686c99deaf228e Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jun 2023 17:52:47 -0400 Subject: [PATCH 128/163] docs: Normalize conf.py --- docs/conf.py | 48 +++++++++++++++++++++++++----------------------- setup.py | 1 - tutorial.ipynb | 4 ++-- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 56adadfa..cdde872a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,40 +1,42 @@ -#!/usr/bin/env python - +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html import os import sys -# Path munging sys.path.insert(0, os.path.abspath('..')) -# Extensions +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'agate' +copyright = '2017, Christopher Groskopf' +version = '1.7.1' +release = version + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx' ] -# autodoc_member_order = 'bysource' -autodoc_default_flags = ['members', 'show-inheritance'] - -intersphinx_mapping = { - 'python': ('https://docs.python.org/3.11', None), - 'leather': ('http://leather.readthedocs.io/en/latest/', None) -} -# Templates templates_path = ['_templates'] -master_doc = 'index' - -# Metadata -project = 'agate' -copyright = '2017, Christopher Groskopf' -version = '1.7.1' -release = version +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] -exclude_patterns = ['_build'] -pygments_style = 'sphinx' +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# HTMl theming html_theme = 'furo' -html_static_path = ['_static'] htmlhelp_basename = 'agatedoc' + +autodoc_default_flags = ['members', 'show-inheritance'] + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'leather': ('http://leather.readthedocs.io/en/latest/', None) +} diff --git a/setup.py b/setup.py index f6541491..deba0bab 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,6 @@ ], 'docs': [ 'Sphinx>=1.2.2', - 'sphinx_rtd_theme>=0.1.6', ], } ) diff --git a/tutorial.ipynb b/tutorial.ipynb index 89ee741e..fc1b7e13 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -197,7 +197,7 @@ "All four of these objects are examples of [`.MappedSequence`](https://agate.readthedocs.io/en/latest/api/columns_and_rows.html#agate.MappedSequence), the foundational type that underlies much of agate's functionality. A MappedSequence functions very similar to a standard Python [`dict`](https://docs.python.org/3/tutorial/datastructures.html#dictionaries), with a few important exceptions:\n", "\n", "* Data may be accessed either by numeric index (e.g. column number) or by a non-integer key (e.g. column name).\n", - "* Items are ordered, just like an instance of [`collections.OrderedDict`](https://docs.python.org/3.5/library/collections.html#collections.OrderedDict).\n", + "* Items are ordered, just like an instance of [`collections.OrderedDict`](https://docs.python.org/3/library/collections.html#collections.OrderedDict).\n", "* Iterating over the sequence returns its *values*, rather than its *keys*.\n", "\n", "To demonstrate the first point, these two lines are both valid ways of getting the first column in the `exonerations` table:" @@ -304,7 +304,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In this case we create our row names using a [`lambda`](https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions) function that takes a row and returns an unique identifer. If your data has a unique column, you can also just pass the column name. (For example, a column of USPS abbrevations or FIPS codes.) Note, however, that your row names can never be `int`, because that is reserved for indexing by numeric order. (A [`decimal.Decimal`](https://docs.python.org/3.5/library/decimal.html#decimal.Decimal) or stringified integer is just fine.)\n", + "In this case we create our row names using a [`lambda`](https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions) function that takes a row and returns an unique identifer. If your data has a unique column, you can also just pass the column name. (For example, a column of USPS abbrevations or FIPS codes.) Note, however, that your row names can never be `int`, because that is reserved for indexing by numeric order. (A [`decimal.Decimal`](https://docs.python.org/3/library/decimal.html#decimal.Decimal) or stringified integer is just fine.)\n", "\n", "Once you've got a specific row, you can then access its individual values (cells, in spreadsheet-speak) either by numeric index or column name:" ] From 00616185f6655419fd0105fb334cab9d73d14220 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jun 2023 18:01:16 -0400 Subject: [PATCH 129/163] docs: Remove csv_py2 --- agate/config.py | 16 ++++++++-------- docs/api/csv.rst | 23 ----------------------- setup.cfg | 4 ---- 3 files changed, 8 insertions(+), 35 deletions(-) diff --git a/agate/config.py b/agate/config.py index c99716b7..4de01307 100644 --- a/agate/config.py +++ b/agate/config.py @@ -12,21 +12,21 @@ +=========================+==========================================+=========================================+ | default_locale | Default locale for number formatting | default_locale('LC_NUMERIC') or 'en_US' | +-------------------------+------------------------------------------+-----------------------------------------+ -| horizontal_line_char | Character to render for horizontal lines | '-' | +| horizontal_line_char | Character to render for horizontal lines | '-' | +-------------------------+------------------------------------------+-----------------------------------------+ -| vertical_line_char | Character to render for vertical lines | '|' | +| vertical_line_char | Character to render for vertical lines | '|' | +-------------------------+------------------------------------------+-----------------------------------------+ -| bar_char | Character to render for bar chart units | '░' | +| bar_char | Character to render for bar chart units | '░' | +-------------------------+------------------------------------------+-----------------------------------------+ -| printable_bar_char | Printable character for bar chart units | ':' | +| printable_bar_char | Printable character for bar chart units | ':' | +-------------------------+------------------------------------------+-----------------------------------------+ -| zero_line_char | Character to render for zero line units | '▓' | +| zero_line_char | Character to render for zero line units | '▓' | +-------------------------+------------------------------------------+-----------------------------------------+ -| printable_zero_line_char| Printable character for zero line units | '|' | +| printable_zero_line_char| Printable character for zero line units | '|' | +-------------------------+------------------------------------------+-----------------------------------------+ -| tick_char | Character to render for axis ticks | '+' | +| tick_char | Character to render for axis ticks | '+' | +-------------------------+------------------------------------------+-----------------------------------------+ -| ellipsis_chars | Characters to render for ellipsis | '...' | +| ellipsis_chars | Characters to render for ellipsis | '...' | +-------------------------+------------------------------------------+-----------------------------------------+ """ diff --git a/docs/api/csv.rst b/docs/api/csv.rst index 4d02128c..59c79771 100644 --- a/docs/api/csv.rst +++ b/docs/api/csv.rst @@ -25,19 +25,6 @@ Python 3 agate.csv_py3.DictReader agate.csv_py3.DictWriter -Python 2 --------- - -.. autosummary:: - :nosignatures: - - agate.csv_py2.reader - agate.csv_py2.writer - agate.csv_py2.Reader - agate.csv_py2.Writer - agate.csv_py2.DictReader - agate.csv_py2.DictWriter - Python 3 details ---------------- @@ -47,13 +34,3 @@ Python 3 details .. autoclass:: agate.csv_py3.Writer .. autoclass:: agate.csv_py3.DictReader .. autoclass:: agate.csv_py3.DictWriter - -Python 2 details ----------------- - -.. autofunction:: agate.csv_py2.reader -.. autofunction:: agate.csv_py2.writer -.. autoclass:: agate.csv_py2.Reader -.. autoclass:: agate.csv_py2.Writer -.. autoclass:: agate.csv_py2.DictReader -.. autoclass:: agate.csv_py2.DictWriter diff --git a/setup.cfg b/setup.cfg index 31cf81cb..9bf57093 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,3 @@ line_length = 119 [bdist_wheel] universal = 1 - -[coverage:run] -omit = - agate/csv_py2.py From ed006840fd28bfe25f2601b5d54484c5d1f20c07 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jun 2023 18:28:02 -0400 Subject: [PATCH 130/163] docs: Use HTTPS URLs --- .github/CONTRIBUTING.md | 6 +++--- CHANGELOG.rst | 6 +++--- README.rst | 2 +- agate/aggregations/mad.py | 2 +- agate/aggregations/max_length.py | 2 +- agate/data_types/date_time.py | 2 +- agate/data_types/number.py | 2 +- docs/about.rst | 4 ++-- docs/conf.py | 2 +- docs/cookbook/charting.rst | 4 ++-- docs/cookbook/compute.rst | 2 +- docs/cookbook/create.rst | 8 ++++---- docs/cookbook/save.rst | 2 +- docs/cookbook/sql.rst | 2 +- docs/cookbook/statistics.rst | 4 ++-- docs/extensions.rst | 14 +++++++------- docs/install.rst | 4 ++-- setup.py | 2 +- tutorial.ipynb | 4 ++-- 19 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 6cff28f2..c36c622d 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -35,8 +35,8 @@ Legalese To the extent that they care, contributors should keep in mind that the source of agate and therefore of any contributions are licensed under the permissive [MIT license]. By submitting a patch or pull request you are agreeing to release your code under this license. You will be acknowledged in the AUTHORS list, the commit history and the hearts and minds of jo - [numpy]: http://www.numpy.org/ - [pandas]: http://pandas.pydata.org/ + [numpy]: https://numpy.org/ + [pandas]: https://pandas.pydata.org/ [GitHub]: https://github.com/wireservice/agate [issue tracker]: https://github.com/wireservice/agate/issues - [MIT license]: http://www.opensource.org/licenses/mit-license.php + [MIT license]: https://opensource.org/license/mit/ diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d7ddb52..6d1ad98a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -103,7 +103,7 @@ This is a minor release fixing several small bugs that were blocking a downstrea 1.5.0 - November 16, 2016 ------------------------- -This release adds SVG charting via the `leather `_ charting library. Charts methods have been added for both :class:`.Table` and :class:`.TableSet`. (The latter create lattice plots.) See the revised tutorial and new cookbook entries for examples. Leather is still an early library. Please `report any bugs `_. +This release adds SVG charting via the `leather `_ charting library. Charts methods have been added for both :class:`.Table` and :class:`.TableSet`. (The latter create lattice plots.) See the revised tutorial and new cookbook entries for examples. Leather is still an early library. Please `report any bugs `_. Also in this release are a :class:`.Slugify` computation and a variety of small fixes and improvements. @@ -117,7 +117,7 @@ The complete list of changes is as follows: * Tables rendered by :meth:`.Table.print_table` are now GitHub Flavored Markdown (GFM) compatible. (#626) * The agate tutorial has been converted to a Jupyter Notebook. * :class:`.Table` now supports ``len`` as a proxy for ``len(table.rows)``. -* Simple SVG charting is now integrated via `leather `_. +* Simple SVG charting is now integrated via `leather `_. * Added :class:`.First` computation. (#634) * :meth:`.Table.print_table` now has a `max_precision` argument to limit Number precision. (#544) * Slug computation now accepts an array of column names to merge. (#617) @@ -366,7 +366,7 @@ This version of agate introduces three major changes. * :const:`.DEFAULT_NULL_VALUES` (the list of strings that mean null) is now importable from ``agate``. * :meth:`.Table.from_csv` and :meth:`.Table.to_csv` are now unicode-safe without separately importing csvkit. * ``agate`` can now be used as a drop-in replacement for Python's ``csv`` module. -* Migrated `csvkit `_'s unicode CSV reading/writing support into agate. (#354) +* Migrated `csvkit `_'s unicode CSV reading/writing support into agate. (#354) 1.0.1 - October 29, 2015 ------------------------ diff --git a/README.rst b/README.rst index 9240443d..558d5d2f 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,6 @@ agate was previously known as journalism. Important links: -* Documentation: http://agate.rtfd.org +* Documentation: https://agate.rtfd.org * Repository: https://github.com/wireservice/agate * Issues: https://github.com/wireservice/agate/issues diff --git a/agate/aggregations/mad.py b/agate/aggregations/mad.py index 8fe9e2ce..59ce595b 100644 --- a/agate/aggregations/mad.py +++ b/agate/aggregations/mad.py @@ -11,7 +11,7 @@ class MAD(Aggregation): """ - Calculate the `median absolute deviation `_ + Calculate the `median absolute deviation `_ of a column. :param column_name: diff --git a/agate/aggregations/max_length.py b/agate/aggregations/max_length.py index ec9146b1..0d65bff1 100644 --- a/agate/aggregations/max_length.py +++ b/agate/aggregations/max_length.py @@ -13,7 +13,7 @@ class MaxLength(Aggregation): Note: On Python 2.7 this function may miscalcuate the length of unicode strings that contain "wide characters". For details see this StackOverflow - answer: http://stackoverflow.com/a/35462951 + answer: https://stackoverflow.com/a/35462951 :param column_name: The name of a column containing :class:`.Text` data. diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index a0ddaf22..b7c3688a 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -18,7 +18,7 @@ class DateTime(DataType): A formatting string for :meth:`datetime.datetime.strptime` to use instead of using regex-based parsing. :param timezone: - A `pytz `_ timezone to apply to each + A `pytz `_ timezone to apply to each parsed date. :param locale: A locale specification such as :code:`en_US` or :code:`de_DE` to use diff --git a/agate/data_types/number.py b/agate/data_types/number.py index b6b73b2d..4bc19e51 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -8,7 +8,7 @@ from agate.data_types.base import DataType from agate.exceptions import CastError -#: A list of currency symbols sourced from `Xe `_. +#: A list of currency symbols sourced from `Xe `_. DEFAULT_CURRENCY_SYMBOLS = ['؋', '$', 'ƒ', '៛', '¥', '₡', '₱', '£', '€', '¢', '﷼', '₪', '₩', '₭', '₮', '₦', '฿', '₤', '₫'] diff --git a/docs/about.rst b/docs/about.rst index e29f8ca6..bc8cbff4 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -11,7 +11,7 @@ Why agate? * Decimal precision everywhere. * Exhaustive user documentation. * Pluggable `extensions `_ that add SQL integration, Excel support, and more. -* Designed with `iPython `_, `Jupyter `_ and `atom/hydrogen `_ in mind. +* Designed with `iPython `_, `Jupyter `_ and `atom/hydrogen `_ in mind. * Pure Python. No C dependencies to compile. * Exhaustive test coverage. * MIT licensed and free for all purposes. @@ -21,7 +21,7 @@ Why agate? Principles ========== -agate is a intended to fill a very particular programming niche. It should not be allowed to become as complex as `numpy `_ or `pandas `_. Please bear in mind the following principles when considering a new feature: +agate is a intended to fill a very particular programming niche. It should not be allowed to become as complex as `numpy `_ or `pandas `_. Please bear in mind the following principles when considering a new feature: * Humans have less time than computers. Optimize for humans. * Most datasets are small. Don't optimize for "big data". diff --git a/docs/conf.py b/docs/conf.py index cdde872a..c50a5a3a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,5 +38,5 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), - 'leather': ('http://leather.readthedocs.io/en/latest/', None) + 'leather': ('https://leather.readthedocs.io/en/latest/', None) } diff --git a/docs/cookbook/charting.rst b/docs/cookbook/charting.rst index f5aa8c47..7b466092 100644 --- a/docs/cookbook/charting.rst +++ b/docs/cookbook/charting.rst @@ -2,7 +2,7 @@ Charts ====== -Agate offers two kinds of built in charting: very simple text bar charts and SVG charting via `leather `_. Both are intended for efficiently exploring data, rather than producing publication-ready charts. +Agate offers two kinds of built in charting: very simple text bar charts and SVG charting via `leather `_. Both are intended for efficiently exploring data, rather than producing publication-ready charts. Text-based bar chart ==================== @@ -119,7 +119,7 @@ SVG lattice chart Using matplotlib ================ -If you need to make more complex charts, you can always use agate with `matplotlib `_. +If you need to make more complex charts, you can always use agate with `matplotlib `_. Here is an example of how you might generate a line chart: diff --git a/docs/cookbook/compute.rst b/docs/cookbook/compute.rst index 725eb76e..d9e0a2aa 100644 --- a/docs/cookbook/compute.rst +++ b/docs/cookbook/compute.rst @@ -198,7 +198,7 @@ The resulting column will contain an integer measuring the edit distance between USA Today Diversity Index ========================= -The `USA Today Diversity Index `_ is a widely cited method for evaluating the racial diversity of a given area. Using a custom :class:`.Computation` makes it simple to calculate. +The `USA Today Diversity Index `_ is a widely cited method for evaluating the racial diversity of a given area. Using a custom :class:`.Computation` makes it simple to calculate. Assuming that your data has a column for the total population, another for the population of each race and a final column for the hispanic population, you can implement the diversity index like this: diff --git a/docs/cookbook/create.rst b/docs/cookbook/create.rst index abafde7a..defa0c82 100644 --- a/docs/cookbook/create.rst +++ b/docs/cookbook/create.rst @@ -139,7 +139,7 @@ From newline-delimited JSON From a SQL database =================== -Use the `agate-sql `_ extension. +Use the `agate-sql `_ extension. .. code-block:: python @@ -152,7 +152,7 @@ Use the `agate-sql `_ extension. From an Excel spreadsheet ========================= -Use the `agate-excel `_ extension. It supports both .xls and .xlsx files. +Use the `agate-excel `_ extension. It supports both .xls and .xlsx files. .. code-block:: python @@ -167,7 +167,7 @@ Use the `agate-excel `_ extension. It suppo From a DBF table ================ -DBF is the file format used to hold tabular data for ArcGIS shapefiles. `agate-dbf `_ extension. +DBF is the file format used to hold tabular data for ArcGIS shapefiles. `agate-dbf `_ extension. .. code-block:: python @@ -180,7 +180,7 @@ DBF is the file format used to hold tabular data for ArcGIS shapefiles. `agate-d From a remote file ================== -Use the `agate-remote `_ extension. +Use the `agate-remote `_ extension. .. code-block:: python diff --git a/docs/cookbook/save.rst b/docs/cookbook/save.rst index 0c7fc4fd..529ab165 100644 --- a/docs/cookbook/save.rst +++ b/docs/cookbook/save.rst @@ -26,7 +26,7 @@ To newline-delimited JSON To a SQL database ================= -Use the `agate-sql `_ extension. +Use the `agate-sql `_ extension. .. code-block:: python diff --git a/docs/cookbook/sql.rst b/docs/cookbook/sql.rst index c85e4ae6..fef2a4b8 100644 --- a/docs/cookbook/sql.rst +++ b/docs/cookbook/sql.rst @@ -6,7 +6,7 @@ agate's command structure is very similar to SQL. The primary difference between .. note:: - All examples in this section use the `PostgreSQL `_ dialect for comparison. + All examples in this section use the `PostgreSQL `_ dialect for comparison. If you want to read and write data from SQL, see :ref:`load_a_table_from_a_sql_database`. diff --git a/docs/cookbook/statistics.rst b/docs/cookbook/statistics.rst index c2fc5643..f6cedb87 100644 --- a/docs/cookbook/statistics.rst +++ b/docs/cookbook/statistics.rst @@ -2,7 +2,7 @@ Statistics ========== -Common descriptive and aggregate statistics are included with the core agate library. For additional statistical methods beyond the scope of agate consider using the `agate-stats `_ extension or integrating with `scipy `_. +Common descriptive and aggregate statistics are included with the core agate library. For additional statistical methods beyond the scope of agate consider using the `agate-stats `_ extension or integrating with `scipy `_. Descriptive statistics ====================== @@ -86,7 +86,7 @@ The output table will be the same format as the previous example, except the val Identify outliers ================= -The `agate-stats `_ extension adds methods for finding outliers. +The `agate-stats `_ extension adds methods for finding outliers. .. code-block:: python diff --git a/docs/extensions.rst b/docs/extensions.rst index 7cf7d8e1..e265d943 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -7,7 +7,7 @@ The core agate library is designed rely on as few dependencies as possible. Howe Using extensions ================ -agate support's plugin-style extensions using a monkey-patching pattern. Libraries can be created that add new methods onto :class:`.Table` and :class:`.TableSet`. For example, `agate-sql `_ adds the ability to read and write tables from a SQL database: +agate support's plugin-style extensions using a monkey-patching pattern. Libraries can be created that add new methods onto :class:`.Table` and :class:`.TableSet`. For example, `agate-sql `_ adds the ability to read and write tables from a SQL database: .. code-block:: python @@ -23,12 +23,12 @@ List of extensions Here is a list of agate extensions that are known to be actively maintained: -* `agate-sql `_: Read and write tables in SQL databases -* `agate-stats `_: Additional statistical methods -* `agate-excel `_: Read excel tables (xls and xlsx) -* `agate-dbf `_: Read dbf tables (from shapefiles) -* `agate-remote `_: Read from remote files -* `agate-lookup `_: Instantly join to hosted `lookup `_ tables. +* `agate-sql `_: Read and write tables in SQL databases +* `agate-stats `_: Additional statistical methods +* `agate-excel `_: Read excel tables (xls and xlsx) +* `agate-dbf `_: Read dbf tables (from shapefiles) +* `agate-remote `_: Read from remote files +* `agate-lookup `_: Instantly join to hosted `lookup `_ tables. Writing your own extensions =========================== diff --git a/docs/install.rst b/docs/install.rst index d86badc6..169f0ece 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -37,10 +37,10 @@ agate supports the following versions of Python: * Python 2.7 * Python 3.5+ -* `PyPy `_ versions >= 4.0.0 +* `PyPy `_ versions >= 4.0.0 It is tested primarily on OSX, but due to its minimal dependencies it should work perfectly on both Linux and Windows. .. note:: - `iPython `_ or `Jupyter `_ user? Agate works great there too. + `iPython `_ or `Jupyter `_ user? Agate works great there too. diff --git a/setup.py b/setup.py index deba0bab..87e401a8 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description_content_type='text/x-rst', author='Christopher Groskopf', author_email='chrisgroskopf@gmail.com', - url='http://agate.readthedocs.org/', + url='https://agate.readthedocs.org/', project_urls={ 'Source': 'https://github.com/wireservice/agate', }, diff --git a/tutorial.ipynb b/tutorial.ipynb index fc1b7e13..cd1b4cf6 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -8,7 +8,7 @@ "\n", "The best way to learn to use any tool is to actually use it. In this tutorial we will use agate to answer some basic questions about a dataset.\n", "\n", - "The data we will be using is a copy of the [National Registry of Exonerations]( http://www.law.umich.edu/special/exoneration/Pages/detaillist.aspx) made on August 28th, 2015. This dataset lists individuals who are known to have been exonerated after having been wrongly convicted in United States courts. At the time this data was copied there were 1,651 entries in the registry." + "The data we will be using is a copy of the [National Registry of Exonerations]( https://www.law.umich.edu/special/exoneration/Pages/detaillist.aspx) made on August 28th, 2015. This dataset lists individuals who are known to have been exonerated after having been wrongly convicted in United States courts. At the time this data was copied there were 1,651 entries in the registry." ] }, { @@ -865,7 +865,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If you find it impossible to believe that an eleven year-old was convicted of murder, I encourage you to read the Registry's [description of the case](http://www.law.umich.edu/special/exoneration/Pages/casedetail.aspx?caseid=3499>).\n", + "If you find it impossible to believe that an eleven year-old was convicted of murder, I encourage you to read the Registry's [description of the case](https://www.law.umich.edu/special/exoneration/Pages/casedetail.aspx?caseid=3499>).\n", "\n", "**Note:** In the previous example we could have omitted the [`Table.limit`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.limit) and passed a ``max_rows=10`` to [`Table.print_table`](https://agate.readthedocs.io/en/latest/api/table.html#agate.Table.print_table) instead. In this case they accomplish exactly the same goal.\n", "\n", From 2627711852558bf408e0f6b4c0592abac60d3ba7 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Jun 2023 19:02:43 -0400 Subject: [PATCH 131/163] test: Relax expectation --- benchmarks/test_joins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 7d27d768..619893ba 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -28,4 +28,4 @@ def test(): min_time = min(results) - self.assertLess(min_time, 10) # CI unreliable, 5s witnessed + self.assertLess(min_time, 15) # CI unreliable, 12s witnessed From a53c2b12452152e61d7c5e5eb0df27e66c266684 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 14 Jun 2023 14:09:08 -0400 Subject: [PATCH 132/163] test: Relax expectation --- benchmarks/test_joins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 619893ba..0c7ad9bb 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -28,4 +28,4 @@ def test(): min_time = min(results) - self.assertLess(min_time, 15) # CI unreliable, 12s witnessed + self.assertLess(min_time, 20) # CI unreliable, 15s witnessed on PyPy From 9b7daa5a1de800d6e18ef6cf72b7e86eca92d298 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:53:43 -0400 Subject: [PATCH 133/163] ci: Add PyPI workflow --- .github/workflows/pypi.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/pypi.yml diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 00000000..2654ebbc --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,23 @@ +name: Publish to PyPI +on: push +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: pip install --upgrade build + - run: python -m build --sdist --wheel + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} From 9375e0d016a75ebb531cde74f6850664e5a128bc Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:52:24 -0400 Subject: [PATCH 134/163] chore: Run pyupgrade --py38-plus **/*.py --- agate/mapped_sequence.py | 2 +- agate/table/__init__.py | 2 +- agate/table/bins.py | 2 +- agate/table/print_bars.py | 4 ++-- agate/table/print_table.py | 2 +- tests/test_py3.py | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index 7878d621..b87f215f 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -65,7 +65,7 @@ def __unicode__(self): if len(self) > 5: sample = '%s, ...' % sample - return ''.format(type(self).__name__, sample) + return f'' def __str__(self): """ diff --git a/agate/table/__init__.py b/agate/table/__init__.py index 8cb3b121..638c6b55 100644 --- a/agate/table/__init__.py +++ b/agate/table/__init__.py @@ -130,7 +130,7 @@ def __init__(self, rows, column_names=None, column_types=None, row_names=None, _ try: row_values.append(cast_funcs[j](d)) except CastError as e: - raise CastError(str(e) + ' Error at row {} column {}.'.format(i, self._column_names[j])) + raise CastError(str(e) + f' Error at row {i} column {self._column_names[j]}.') new_rows.append(Row(row_values, self._column_names)) else: diff --git a/agate/table/bins.py b/agate/table/bins.py index 1f35c3cd..78608792 100644 --- a/agate/table/bins.py +++ b/agate/table/bins.py @@ -63,7 +63,7 @@ def name_bin(i, j, first_exclusive=True, last_exclusive=False): exclusive = format_decimal(j, format=break_formatter) output = '[' if first_exclusive else '(' - output += '{} - {}'.format(inclusive, exclusive) + output += f'{inclusive} - {exclusive}' output += ']' if last_exclusive else ')' return output diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index 23f89348..25dc3a2d 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -175,7 +175,7 @@ def write(line): output.write(line + '\n') # Chart top - top_line = '{} {}'.format(y_label.ljust(max_label_width), x_label.rjust(max_value_width)) + top_line = f'{y_label.ljust(max_label_width)} {x_label.rjust(max_value_width)}' write(top_line) # Bars @@ -210,7 +210,7 @@ def write(line): bar = bar.ljust(plot_width) - write('{} {} {}'.format(label_text, value_text, bar)) + write(f'{label_text} {value_text} {bar}') # Axis & ticks axis = horizontal_line * plot_width diff --git a/agate/table/print_table.py b/agate/table/print_table.py index 0ad5c5af..6d502f5a 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -137,7 +137,7 @@ def write_row(formatted_row): text = v_line.join(row_output) - write('{}{}{}'.format(v_line, text, v_line)) + write(f'{v_line}{text}{v_line}') divider = '{v_line} {columns} {v_line}'.format( v_line=v_line, diff --git a/tests/test_py3.py b/tests/test_py3.py index f3a561a5..c50f4ea1 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -240,5 +240,5 @@ def test_sniffer(self): actual = csv_py3.Sniffer().sniff(contents).__dict__ expected = csv.Sniffer().sniff(contents).__dict__ - self.assertEqual(direct, expected, '{!r} != {!r}'.format(direct, expected)) - self.assertEqual(actual, expected, '{!r} != {!r}'.format(actual, expected)) + self.assertEqual(direct, expected, f'{direct!r} != {expected!r}') + self.assertEqual(actual, expected, f'{actual!r} != {expected!r}') From 0acba502d95d28aeb8c28b16279c132107bc5681 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:11:04 -0400 Subject: [PATCH 135/163] test: Remove shebang. chore: Remove docs extras. --- setup.py | 3 --- tests/test_agate.py | 2 -- tests/test_aggregations.py | 2 -- tests/test_columns.py | 2 -- tests/test_data_types.py | 2 -- tests/test_fixed.py | 2 -- tests/test_from_json.py | 2 -- tests/test_mapped_sequence.py | 2 -- tests/test_py3.py | 2 -- tests/test_table/__init__.py | 2 -- tests/test_table/test_aggregate.py | 2 -- tests/test_table/test_bins.py | 2 -- tests/test_table/test_charting.py | 2 -- tests/test_table/test_compute.py | 2 -- tests/test_table/test_denormalize.py | 2 -- tests/test_table/test_from_csv.py | 2 -- tests/test_table/test_from_fixed.py | 2 -- tests/test_table/test_group_by.py | 2 -- tests/test_table/test_homogenize.py | 2 -- tests/test_table/test_join.py | 2 -- tests/test_table/test_merge.py | 2 -- tests/test_table/test_normalize.py | 2 -- tests/test_table/test_order_py.py | 2 -- tests/test_table/test_pivot.py | 2 -- tests/test_table/test_print_bars.py | 2 -- tests/test_table/test_print_html.py | 2 -- tests/test_table/test_print_table.py | 2 -- tests/test_table/test_rename.py | 2 -- tests/test_table/test_to_csv.py | 2 -- tests/test_table/test_to_json.py | 2 -- tests/test_tableset/__init__.py | 2 -- tests/test_tableset/test_aggregate.py | 2 -- tests/test_tableset/test_charting.py | 2 -- tests/test_tableset/test_having.py | 2 -- tests/test_tableset/test_merge.py | 2 -- tests/test_type_tester.py | 2 -- tests/test_utils.py | 2 -- 37 files changed, 75 deletions(-) diff --git a/setup.py b/setup.py index 87e401a8..3aa57f42 100644 --- a/setup.py +++ b/setup.py @@ -56,8 +56,5 @@ 'pytest-cov', 'pytz>=2015.4', ], - 'docs': [ - 'Sphinx>=1.2.2', - ], } ) diff --git a/tests/test_agate.py b/tests/test_agate.py index 41fbe7d7..78a77da7 100644 --- a/tests/test_agate.py +++ b/tests/test_agate.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest import agate diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index e1645ed7..c7fd352a 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import datetime import sys import unittest diff --git a/tests/test_columns.py b/tests/test_columns.py index cec1b8f2..3ff3f4b0 100644 --- a/tests/test_columns.py +++ b/tests/test_columns.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import pickle import unittest from decimal import Decimal diff --git a/tests/test_data_types.py b/tests/test_data_types.py index ae4b7de2..3ac1a7e2 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import datetime import pickle import unittest diff --git a/tests/test_fixed.py b/tests/test_fixed.py index 18973d6e..22a95da5 100644 --- a/tests/test_fixed.py +++ b/tests/test_fixed.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest from agate import csv, fixed diff --git a/tests/test_from_json.py b/tests/test_from_json.py index a99a2173..2dcec013 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta diff --git a/tests/test_mapped_sequence.py b/tests/test_mapped_sequence.py index b2ec1d3f..c483c309 100644 --- a/tests/test_mapped_sequence.py +++ b/tests/test_mapped_sequence.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest from agate.mapped_sequence import MappedSequence diff --git a/tests/test_py3.py b/tests/test_py3.py index c50f4ea1..de24373c 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import csv import os import unittest diff --git a/tests/test_table/__init__.py b/tests/test_table/__init__.py index 4b582dad..ac9dacad 100644 --- a/tests/test_table/__init__.py +++ b/tests/test_table/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import warnings from decimal import Decimal diff --git a/tests/test_table/test_aggregate.py b/tests/test_table/test_aggregate.py index 2052b363..a8f7a180 100644 --- a/tests/test_table/test_aggregate.py +++ b/tests/test_table/test_aggregate.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.aggregations import Count, Sum from agate.data_types import Number, Text diff --git a/tests/test_table/test_bins.py b/tests/test_table/test_bins.py index f3395b23..3f2f68eb 100644 --- a/tests/test_table/test_bins.py +++ b/tests/test_table/test_bins.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from decimal import Decimal from babel.numbers import get_decimal_symbol diff --git a/tests/test_table/test_charting.py b/tests/test_table/test_charting.py index 2663a1fa..9656f80c 100644 --- a/tests/test_table/test_charting.py +++ b/tests/test_table/test_charting.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import leather from agate import Table diff --git a/tests/test_table/test_compute.py b/tests/test_table/test_compute.py index 42e4b409..530dc12b 100644 --- a/tests/test_table/test_compute.py +++ b/tests/test_table/test_compute.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.computations import Formula diff --git a/tests/test_table/test_denormalize.py b/tests/test_table/test_denormalize.py index a27ab0e7..05d80564 100644 --- a/tests/test_table/test_denormalize.py +++ b/tests/test_table/test_denormalize.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_from_csv.py b/tests/test_table/test_from_csv.py index 2384849a..d4cbc693 100644 --- a/tests/test_table/test_from_csv.py +++ b/tests/test_table/test_from_csv.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import warnings from agate import Table diff --git a/tests/test_table/test_from_fixed.py b/tests/test_table/test_from_fixed.py index 914fb72c..a546125a 100644 --- a/tests/test_table/test_from_fixed.py +++ b/tests/test_table/test_from_fixed.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_group_by.py b/tests/test_table/test_group_by.py index 3b0e278b..cb51ffd4 100644 --- a/tests/test_table/test_group_by.py +++ b/tests/test_table/test_group_by.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from decimal import Decimal from agate import Table, TableSet diff --git a/tests/test_table/test_homogenize.py b/tests/test_table/test_homogenize.py index 1183aa06..fb2ab751 100644 --- a/tests/test_table/test_homogenize.py +++ b/tests/test_table/test_homogenize.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_join.py b/tests/test_table/test_join.py index 763ca503..1dd87e53 100644 --- a/tests/test_table/test_join.py +++ b/tests/test_table/test_join.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_merge.py b/tests/test_table/test_merge.py index b9d823f4..f23f9f50 100644 --- a/tests/test_table/test_merge.py +++ b/tests/test_table/test_merge.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.data_types import Number, Text from agate.exceptions import DataTypeError diff --git a/tests/test_table/test_normalize.py b/tests/test_table/test_normalize.py index 4ba0252d..a7bab9c2 100644 --- a/tests/test_table/test_normalize.py +++ b/tests/test_table/test_normalize.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_order_py.py b/tests/test_table/test_order_py.py index 6692a025..e7f4e0a6 100644 --- a/tests/test_table/test_order_py.py +++ b/tests/test_table/test_order_py.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import Table from agate.data_types import Number, Text from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_pivot.py b/tests/test_table/test_pivot.py index dc67f926..c401fa93 100644 --- a/tests/test_table/test_pivot.py +++ b/tests/test_table/test_pivot.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import sys from decimal import Decimal diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index d80ad590..ea149b73 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from io import StringIO from babel.numbers import format_decimal diff --git a/tests/test_table/test_print_html.py b/tests/test_table/test_print_html.py index 3e9cab47..10833d08 100644 --- a/tests/test_table/test_print_html.py +++ b/tests/test_table/test_print_html.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import warnings from html.parser import HTMLParser from io import StringIO diff --git a/tests/test_table/test_print_table.py b/tests/test_table/test_print_table.py index 5061e5a3..e0cb9b51 100644 --- a/tests/test_table/test_print_table.py +++ b/tests/test_table/test_print_table.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from io import StringIO from babel.numbers import get_decimal_symbol diff --git a/tests/test_table/test_rename.py b/tests/test_table/test_rename.py index 2ddc6f84..16f71b2c 100644 --- a/tests/test_table/test_rename.py +++ b/tests/test_table/test_rename.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import warnings from agate import Table diff --git a/tests/test_table/test_to_csv.py b/tests/test_table/test_to_csv.py index 11ace106..766d82c9 100644 --- a/tests/test_table/test_to_csv.py +++ b/tests/test_table/test_to_csv.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import sys from io import StringIO diff --git a/tests/test_table/test_to_json.py b/tests/test_table/test_to_json.py index 023d3f4f..ad023a44 100644 --- a/tests/test_table/test_to_json.py +++ b/tests/test_table/test_to_json.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import json import os import sys diff --git a/tests/test_tableset/__init__.py b/tests/test_tableset/__init__.py index e3b30715..8815bf60 100644 --- a/tests/test_tableset/__init__.py +++ b/tests/test_tableset/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import json import shutil from collections import OrderedDict diff --git a/tests/test_tableset/test_aggregate.py b/tests/test_tableset/test_aggregate.py index f423191f..c19aa0b4 100644 --- a/tests/test_tableset/test_aggregate.py +++ b/tests/test_tableset/test_aggregate.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from collections import OrderedDict from decimal import Decimal diff --git a/tests/test_tableset/test_charting.py b/tests/test_tableset/test_charting.py index 90758e34..7eb459f3 100644 --- a/tests/test_tableset/test_charting.py +++ b/tests/test_tableset/test_charting.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from collections import OrderedDict import leather diff --git a/tests/test_tableset/test_having.py b/tests/test_tableset/test_having.py index 83d5ad9b..19d5bcd8 100644 --- a/tests/test_tableset/test_having.py +++ b/tests/test_tableset/test_having.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from collections import OrderedDict from agate import Table, TableSet diff --git a/tests/test_tableset/test_merge.py b/tests/test_tableset/test_merge.py index bc1ee533..196ae00e 100644 --- a/tests/test_tableset/test_merge.py +++ b/tests/test_tableset/test_merge.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from collections import OrderedDict from agate import Table, TableSet diff --git a/tests/test_type_tester.py b/tests/test_type_tester.py index 4dde9b06..43af7476 100644 --- a/tests/test_type_tester.py +++ b/tests/test_type_tester.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta diff --git a/tests/test_utils.py b/tests/test_utils.py index 9dae3b6a..f73b69f6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest from decimal import Decimal From 1d68d33b35843eed81042ba765c9247725dbbad1 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:24:53 -0400 Subject: [PATCH 136/163] docs: Remove patch() from cookbook --- docs/cookbook/create.rst | 8 -------- docs/cookbook/lookup.rst | 2 -- docs/cookbook/statistics.rst | 2 -- 3 files changed, 12 deletions(-) diff --git a/docs/cookbook/create.rst b/docs/cookbook/create.rst index defa0c82..534e6ce4 100644 --- a/docs/cookbook/create.rst +++ b/docs/cookbook/create.rst @@ -145,8 +145,6 @@ Use the `agate-sql `_ extension. import agatesql - agatesql.patch() - table = agate.Table.from_sql('postgresql:///database', 'input_table') From an Excel spreadsheet @@ -158,8 +156,6 @@ Use the `agate-excel `_ extension. It supp import agateexcel - agateexcel.patch() - table = agate.Table.from_xls('test.xls', sheet='data') table2 = agate.Table.from_xlsx('test.xlsx', sheet='data') @@ -173,8 +169,6 @@ DBF is the file format used to hold tabular data for ArcGIS shapefiles. `agate-d import agatedbf - agatedbf.patch() - table = agate.Table.from_dbf('test.dbf') From a remote file @@ -187,8 +181,6 @@ Use the `agate-remote `_ extension. import agateremote - agateremote.patch() - table = agate.Table.from_url('https://raw.githubusercontent.com/wireservice/agate/master/examples/test.csv') agate-remote also let’s you create an Archive, which is a reference to a group of tables with a known path structure. diff --git a/docs/cookbook/lookup.rst b/docs/cookbook/lookup.rst index cb58e5de..4fe1c82e 100644 --- a/docs/cookbook/lookup.rst +++ b/docs/cookbook/lookup.rst @@ -33,8 +33,6 @@ We can map the ``year`` column to its annual CPI index in one lookup call. import agatelookup - agatelookup.patch() - join_year_cpi = table.lookup('year', 'cpi') The return table will have now have a new column: diff --git a/docs/cookbook/statistics.rst b/docs/cookbook/statistics.rst index f6cedb87..7208cc1d 100644 --- a/docs/cookbook/statistics.rst +++ b/docs/cookbook/statistics.rst @@ -92,8 +92,6 @@ The `agate-stats `_ extension adds methods for fi import agatestats - agatestats.patch() - outliers = table.stdev_outliers('salary', deviations=3, reject=False) By specifying :code:`reject=True` you can instead return a table including only those values **not** identified as outliers. From 0432b7cdcdd833b5b3a7f816a5a2ae5683a4fa87 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:35:12 -0400 Subject: [PATCH 137/163] chore: Remove shebang --- agate/__init__.py | 2 -- agate/aggregations/__init__.py | 2 -- agate/aggregations/all.py | 2 -- agate/aggregations/any.py | 2 -- agate/aggregations/base.py | 2 -- agate/aggregations/count.py | 2 -- agate/aggregations/deciles.py | 2 -- agate/aggregations/first.py | 2 -- agate/aggregations/has_nulls.py | 2 -- agate/aggregations/iqr.py | 2 -- agate/aggregations/mad.py | 2 -- agate/aggregations/max.py | 2 -- agate/aggregations/max_length.py | 2 -- agate/aggregations/max_precision.py | 2 -- agate/aggregations/mean.py | 2 -- agate/aggregations/median.py | 2 -- agate/aggregations/min.py | 2 -- agate/aggregations/mode.py | 2 -- agate/aggregations/percentiles.py | 2 -- agate/aggregations/quartiles.py | 2 -- agate/aggregations/quintiles.py | 2 -- agate/aggregations/stdev.py | 2 -- agate/aggregations/summary.py | 2 -- agate/aggregations/variance.py | 2 -- agate/columns.py | 2 -- agate/computations/__init__.py | 2 -- agate/computations/base.py | 2 -- agate/computations/change.py | 2 -- agate/computations/formula.py | 2 -- agate/computations/percent.py | 2 -- agate/computations/percent_change.py | 2 -- agate/computations/percentile_rank.py | 2 -- agate/computations/rank.py | 2 -- agate/computations/slug.py | 2 -- agate/config.py | 2 -- agate/csv_py3.py | 2 -- agate/data_types/__init__.py | 2 -- agate/data_types/base.py | 2 -- agate/data_types/boolean.py | 2 -- agate/data_types/date.py | 2 -- agate/data_types/date_time.py | 2 -- agate/data_types/number.py | 2 -- agate/data_types/text.py | 2 -- agate/data_types/time_delta.py | 2 -- agate/exceptions.py | 2 -- agate/fixed.py | 2 -- agate/mapped_sequence.py | 2 -- agate/rows.py | 2 -- agate/table/__init__.py | 2 -- agate/table/from_csv.py | 2 -- agate/table/from_fixed.py | 2 -- agate/table/from_json.py | 2 -- agate/table/from_object.py | 2 -- agate/tableset/__init__.py | 2 -- agate/tableset/from_csv.py | 2 -- agate/tableset/from_json.py | 2 -- agate/tableset/print_structure.py | 2 -- agate/tableset/proxy_methods.py | 2 -- agate/tableset/to_csv.py | 2 -- agate/tableset/to_json.py | 2 -- agate/testcase.py | 2 -- agate/type_tester.py | 2 -- agate/utils.py | 2 -- agate/warns.py | 2 -- 64 files changed, 128 deletions(-) diff --git a/agate/__init__.py b/agate/__init__.py index ae6b4b2b..339fe694 100644 --- a/agate/__init__.py +++ b/agate/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import agate.csv_py3 as csv from agate.aggregations import * diff --git a/agate/aggregations/__init__.py b/agate/aggregations/__init__.py index 01d9a9cc..6d2cc2f8 100644 --- a/agate/aggregations/__init__.py +++ b/agate/aggregations/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ Aggregations create a new value by summarizing a :class:`.Column`. For example, :class:`.Mean`, when applied to a column containing :class:`.Number` diff --git a/agate/aggregations/all.py b/agate/aggregations/all.py index f4ecc149..0915e473 100644 --- a/agate/aggregations/all.py +++ b/agate/aggregations/all.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.data_types import Boolean diff --git a/agate/aggregations/any.py b/agate/aggregations/any.py index 30ddd7e9..de25db97 100644 --- a/agate/aggregations/any.py +++ b/agate/aggregations/any.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.data_types import Boolean diff --git a/agate/aggregations/base.py b/agate/aggregations/base.py index 7a440897..aa7b48d8 100644 --- a/agate/aggregations/base.py +++ b/agate/aggregations/base.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.exceptions import UnsupportedAggregationError diff --git a/agate/aggregations/count.py b/agate/aggregations/count.py index f3af86fe..7b6f8336 100644 --- a/agate/aggregations/count.py +++ b/agate/aggregations/count.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.data_types import Number from agate.utils import default diff --git a/agate/aggregations/deciles.py b/agate/aggregations/deciles.py index 188ee5f7..7d63233c 100644 --- a/agate/aggregations/deciles.py +++ b/agate/aggregations/deciles.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.percentiles import Percentiles diff --git a/agate/aggregations/first.py b/agate/aggregations/first.py index 32123b41..df774b1e 100644 --- a/agate/aggregations/first.py +++ b/agate/aggregations/first.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation diff --git a/agate/aggregations/has_nulls.py b/agate/aggregations/has_nulls.py index 6f464932..039f480a 100644 --- a/agate/aggregations/has_nulls.py +++ b/agate/aggregations/has_nulls.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.data_types import Boolean diff --git a/agate/aggregations/iqr.py b/agate/aggregations/iqr.py index cc902f50..12aa8c56 100644 --- a/agate/aggregations/iqr.py +++ b/agate/aggregations/iqr.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.percentiles import Percentiles diff --git a/agate/aggregations/mad.py b/agate/aggregations/mad.py index 59ce595b..52958639 100644 --- a/agate/aggregations/mad.py +++ b/agate/aggregations/mad.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.median import Median diff --git a/agate/aggregations/max.py b/agate/aggregations/max.py index e79d1d8e..f769ee43 100644 --- a/agate/aggregations/max.py +++ b/agate/aggregations/max.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.data_types import Date, DateTime, Number, TimeDelta from agate.exceptions import DataTypeError diff --git a/agate/aggregations/max_length.py b/agate/aggregations/max_length.py index 0d65bff1..33595a80 100644 --- a/agate/aggregations/max_length.py +++ b/agate/aggregations/max_length.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from decimal import Decimal from agate.aggregations.base import Aggregation diff --git a/agate/aggregations/max_precision.py b/agate/aggregations/max_precision.py index 2aa6e5ff..a2dc9095 100644 --- a/agate/aggregations/max_precision.py +++ b/agate/aggregations/max_precision.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.data_types import Number from agate.exceptions import DataTypeError diff --git a/agate/aggregations/mean.py b/agate/aggregations/mean.py index 689b674f..107466f3 100644 --- a/agate/aggregations/mean.py +++ b/agate/aggregations/mean.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.sum import Sum diff --git a/agate/aggregations/median.py b/agate/aggregations/median.py index 5abf4e02..aa68d28d 100644 --- a/agate/aggregations/median.py +++ b/agate/aggregations/median.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.percentiles import Percentiles diff --git a/agate/aggregations/min.py b/agate/aggregations/min.py index a4161fe9..2130739a 100644 --- a/agate/aggregations/min.py +++ b/agate/aggregations/min.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.data_types import Date, DateTime, Number, TimeDelta from agate.exceptions import DataTypeError diff --git a/agate/aggregations/mode.py b/agate/aggregations/mode.py index 0deb9295..8ea50b5d 100644 --- a/agate/aggregations/mode.py +++ b/agate/aggregations/mode.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from collections import defaultdict from agate.aggregations.base import Aggregation diff --git a/agate/aggregations/percentiles.py b/agate/aggregations/percentiles.py index 1a7c27c4..6fb48d0d 100644 --- a/agate/aggregations/percentiles.py +++ b/agate/aggregations/percentiles.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import math from agate.aggregations.base import Aggregation diff --git a/agate/aggregations/quartiles.py b/agate/aggregations/quartiles.py index 7025056f..103c4027 100644 --- a/agate/aggregations/quartiles.py +++ b/agate/aggregations/quartiles.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.percentiles import Percentiles diff --git a/agate/aggregations/quintiles.py b/agate/aggregations/quintiles.py index 05bed638..8ded98ee 100644 --- a/agate/aggregations/quintiles.py +++ b/agate/aggregations/quintiles.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.percentiles import Percentiles diff --git a/agate/aggregations/stdev.py b/agate/aggregations/stdev.py index 1b27c421..398d3272 100644 --- a/agate/aggregations/stdev.py +++ b/agate/aggregations/stdev.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.variance import PopulationVariance, Variance diff --git a/agate/aggregations/summary.py b/agate/aggregations/summary.py index 1ae26f24..e1ab4a43 100644 --- a/agate/aggregations/summary.py +++ b/agate/aggregations/summary.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation diff --git a/agate/aggregations/variance.py b/agate/aggregations/variance.py index 0cdb7a93..f81857eb 100644 --- a/agate/aggregations/variance.py +++ b/agate/aggregations/variance.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.mean import Mean diff --git a/agate/columns.py b/agate/columns.py index 3b99307a..7aa8e365 100644 --- a/agate/columns.py +++ b/agate/columns.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains the :class:`Column` class, which defines a "vertical" array of tabular data. Whereas :class:`.Row` instances are independent of their diff --git a/agate/computations/__init__.py b/agate/computations/__init__.py index 3639215e..f8232faf 100644 --- a/agate/computations/__init__.py +++ b/agate/computations/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ Computations create a new value for each :class:`.Row` in a :class:`.Table`. When used with :meth:`.Table.compute` these new values become a new column. diff --git a/agate/computations/base.py b/agate/computations/base.py index 7de4463c..69577aed 100644 --- a/agate/computations/base.py +++ b/agate/computations/base.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - class Computation: # pragma: no cover """ diff --git a/agate/computations/change.py b/agate/computations/change.py index 5149d229..ea03fe3f 100644 --- a/agate/computations/change.py +++ b/agate/computations/change.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.has_nulls import HasNulls from agate.computations.base import Computation from agate.data_types import Date, DateTime, Number, TimeDelta diff --git a/agate/computations/formula.py b/agate/computations/formula.py index 3a0f947b..3f5d66de 100644 --- a/agate/computations/formula.py +++ b/agate/computations/formula.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.computations.base import Computation diff --git a/agate/computations/percent.py b/agate/computations/percent.py index 422ba4bd..da3a20ff 100644 --- a/agate/computations/percent.py +++ b/agate/computations/percent.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.has_nulls import HasNulls from agate.aggregations.sum import Sum diff --git a/agate/computations/percent_change.py b/agate/computations/percent_change.py index 3a2dd485..49f905ee 100644 --- a/agate/computations/percent_change.py +++ b/agate/computations/percent_change.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.has_nulls import HasNulls from agate.computations.base import Computation from agate.data_types import Number diff --git a/agate/computations/percentile_rank.py b/agate/computations/percentile_rank.py index e3c912ef..667c7064 100644 --- a/agate/computations/percentile_rank.py +++ b/agate/computations/percentile_rank.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.percentiles import Percentiles from agate.computations.rank import Rank from agate.data_types import Number diff --git a/agate/computations/rank.py b/agate/computations/rank.py index a6fe88b2..e525b94f 100644 --- a/agate/computations/rank.py +++ b/agate/computations/rank.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from decimal import Decimal from functools import cmp_to_key diff --git a/agate/computations/slug.py b/agate/computations/slug.py index 68a00fd4..d8b3339f 100644 --- a/agate/computations/slug.py +++ b/agate/computations/slug.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.aggregations.has_nulls import HasNulls from agate.computations.base import Computation from agate.data_types import Text diff --git a/agate/config.py b/agate/config.py index 4de01307..b20b275e 100644 --- a/agate/config.py +++ b/agate/config.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains the global configuration for agate. Users should use :meth:`get_option` and :meth:`set_option` to modify the global diff --git a/agate/csv_py3.py b/agate/csv_py3.py index 79b8cc8b..a408924c 100644 --- a/agate/csv_py3.py +++ b/agate/csv_py3.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains the Python 3 replacement for :mod:`csv`. """ diff --git a/agate/data_types/__init__.py b/agate/data_types/__init__.py index 6b91e278..0bb3d1d8 100644 --- a/agate/data_types/__init__.py +++ b/agate/data_types/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ Data types define how data should be imported during the creation of a :class:`.Table`. diff --git a/agate/data_types/base.py b/agate/data_types/base.py index fe583028..fc9ff4b4 100644 --- a/agate/data_types/base.py +++ b/agate/data_types/base.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.exceptions import CastError diff --git a/agate/data_types/boolean.py b/agate/data_types/boolean.py index 8ad67cda..63137a3e 100644 --- a/agate/data_types/boolean.py +++ b/agate/data_types/boolean.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from decimal import Decimal from agate.data_types.base import DEFAULT_NULL_VALUES, DataType diff --git a/agate/data_types/date.py b/agate/data_types/date.py index 639cacbe..4476bbd7 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import locale from datetime import date, datetime, time diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index b7c3688a..a4e8a661 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import datetime import locale diff --git a/agate/data_types/number.py b/agate/data_types/number.py index 4bc19e51..bc29fda6 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import warnings from decimal import Decimal, InvalidOperation diff --git a/agate/data_types/text.py b/agate/data_types/text.py index be5f5711..178e43cc 100644 --- a/agate/data_types/text.py +++ b/agate/data_types/text.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate.data_types.base import DataType diff --git a/agate/data_types/time_delta.py b/agate/data_types/time_delta.py index 49ccbeed..0c0fc73d 100644 --- a/agate/data_types/time_delta.py +++ b/agate/data_types/time_delta.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import datetime import pytimeparse diff --git a/agate/exceptions.py b/agate/exceptions.py index faaaee70..6bf4294f 100644 --- a/agate/exceptions.py +++ b/agate/exceptions.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains various exceptions raised by agate. """ diff --git a/agate/fixed.py b/agate/fixed.py index 6b9e12ae..42fd89e7 100644 --- a/agate/fixed.py +++ b/agate/fixed.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains a generic parser for fixed-width files. It operates similar to Python's built-in CSV reader. diff --git a/agate/mapped_sequence.py b/agate/mapped_sequence.py index b87f215f..bb1d16c5 100644 --- a/agate/mapped_sequence.py +++ b/agate/mapped_sequence.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains the :class:`MappedSequence` class that forms the foundation for agate's :class:`.Row` and :class:`.Column` as well as for named sequences of diff --git a/agate/rows.py b/agate/rows.py index 431118d2..5b0bf0cc 100644 --- a/agate/rows.py +++ b/agate/rows.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains agate's :class:`Row` implementation. Rows are independent of both the :class:`.Table` that contains them as well as the :class:`.Columns` diff --git a/agate/table/__init__.py b/agate/table/__init__.py index 638c6b55..51afca46 100644 --- a/agate/table/__init__.py +++ b/agate/table/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ The :class:`.Table` object is the most important class in agate. Tables are created by supplying row data, column names and subclasses of :class:`.DataType` diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index b8a6297a..3b1ac074 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import itertools from io import StringIO diff --git a/agate/table/from_fixed.py b/agate/table/from_fixed.py index 0e2653b6..08ad9a0e 100644 --- a/agate/table/from_fixed.py +++ b/agate/table/from_fixed.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import fixed, utils diff --git a/agate/table/from_json.py b/agate/table/from_json.py index eacb8618..77a4062d 100644 --- a/agate/table/from_json.py +++ b/agate/table/from_json.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import json from collections import OrderedDict from decimal import Decimal diff --git a/agate/table/from_object.py b/agate/table/from_object.py index 5675d7a9..f5357f05 100644 --- a/agate/table/from_object.py +++ b/agate/table/from_object.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from agate import utils diff --git a/agate/tableset/__init__.py b/agate/tableset/__init__.py index 972578af..4876d00b 100644 --- a/agate/tableset/__init__.py +++ b/agate/tableset/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ The :class:`.TableSet` class collects a set of related tables in a single data structure. The most common way of creating a :class:`.TableSet` is using the diff --git a/agate/tableset/from_csv.py b/agate/tableset/from_csv.py index cde22bdb..7549289d 100644 --- a/agate/tableset/from_csv.py +++ b/agate/tableset/from_csv.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os from collections import OrderedDict from glob import glob diff --git a/agate/tableset/from_json.py b/agate/tableset/from_json.py index bb79074f..209b39b6 100644 --- a/agate/tableset/from_json.py +++ b/agate/tableset/from_json.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import json import os from collections import OrderedDict diff --git a/agate/tableset/print_structure.py b/agate/tableset/print_structure.py index f52b2e53..e8422494 100644 --- a/agate/tableset/print_structure.py +++ b/agate/tableset/print_structure.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import sys from agate.data_types import Text diff --git a/agate/tableset/proxy_methods.py b/agate/tableset/proxy_methods.py index b8207cde..4b613c50 100644 --- a/agate/tableset/proxy_methods.py +++ b/agate/tableset/proxy_methods.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - def bins(self, *args, **kwargs): """ diff --git a/agate/tableset/to_csv.py b/agate/tableset/to_csv.py index 7c268b5b..b3536fc3 100644 --- a/agate/tableset/to_csv.py +++ b/agate/tableset/to_csv.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os diff --git a/agate/tableset/to_json.py b/agate/tableset/to_json.py index dea7916c..67b5e647 100644 --- a/agate/tableset/to_json.py +++ b/agate/tableset/to_json.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import json import os from collections import OrderedDict diff --git a/agate/testcase.py b/agate/testcase.py index 71db94ee..72aa228a 100644 --- a/agate/testcase.py +++ b/agate/testcase.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest import agate diff --git a/agate/type_tester.py b/agate/type_tester.py index 3c82210a..adcc3472 100644 --- a/agate/type_tester.py +++ b/agate/type_tester.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import warnings from copy import copy diff --git a/agate/utils.py b/agate/utils.py index e95d256f..fc290c18 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ This module contains a collection of utility classes and functions used in agate. diff --git a/agate/warns.py b/agate/warns.py index 53cd408d..b5eacc74 100644 --- a/agate/warns.py +++ b/agate/warns.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import warnings From 160cc84ecc2027ea0dc19a9ece0441c4c4a63260 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:35:57 -0400 Subject: [PATCH 138/163] chore: Remove shebang + pylint --- agate/table/aggregate.py | 3 --- agate/table/bar_chart.py | 3 --- agate/table/bins.py | 3 --- agate/table/column_chart.py | 3 --- agate/table/compute.py | 3 --- agate/table/denormalize.py | 3 --- agate/table/distinct.py | 3 --- agate/table/exclude.py | 3 --- agate/table/find.py | 3 --- agate/table/group_by.py | 3 --- agate/table/homogenize.py | 3 --- agate/table/join.py | 3 --- agate/table/limit.py | 3 --- agate/table/line_chart.py | 3 --- agate/table/merge.py | 3 --- agate/table/normalize.py | 3 --- agate/table/order_by.py | 3 --- agate/table/pivot.py | 3 --- agate/table/print_bars.py | 3 --- agate/table/print_html.py | 3 --- agate/table/print_structure.py | 3 --- agate/table/print_table.py | 3 --- agate/table/rename.py | 3 --- agate/table/scatterplot.py | 3 --- agate/table/select.py | 3 --- agate/table/to_csv.py | 3 --- agate/table/to_json.py | 3 --- agate/table/where.py | 3 --- agate/tableset/aggregate.py | 3 --- agate/tableset/bar_chart.py | 3 --- agate/tableset/column_chart.py | 3 --- agate/tableset/having.py | 3 --- agate/tableset/line_chart.py | 3 --- agate/tableset/merge.py | 3 --- agate/tableset/scatterplot.py | 3 --- 35 files changed, 105 deletions(-) diff --git a/agate/table/aggregate.py b/agate/table/aggregate.py index 66878513..e5071092 100644 --- a/agate/table/aggregate.py +++ b/agate/table/aggregate.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from collections import OrderedDict from agate import utils diff --git a/agate/table/bar_chart.py b/agate/table/bar_chart.py index 2853dda1..2c7ec636 100644 --- a/agate/table/bar_chart.py +++ b/agate/table/bar_chart.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather diff --git a/agate/table/bins.py b/agate/table/bins.py index 78608792..3514877f 100644 --- a/agate/table/bins.py +++ b/agate/table/bins.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from decimal import Decimal from babel.numbers import format_decimal diff --git a/agate/table/column_chart.py b/agate/table/column_chart.py index 5ea31cf1..c2467ee8 100644 --- a/agate/table/column_chart.py +++ b/agate/table/column_chart.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather diff --git a/agate/table/compute.py b/agate/table/compute.py index 6260ecf5..11b94c88 100644 --- a/agate/table/compute.py +++ b/agate/table/compute.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from collections import OrderedDict from copy import copy diff --git a/agate/table/denormalize.py b/agate/table/denormalize.py index 1a571261..8175187b 100644 --- a/agate/table/denormalize.py +++ b/agate/table/denormalize.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from collections import OrderedDict from decimal import Decimal diff --git a/agate/table/distinct.py b/agate/table/distinct.py index a991bc2e..9f510967 100644 --- a/agate/table/distinct.py +++ b/agate/table/distinct.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils diff --git a/agate/table/exclude.py b/agate/table/exclude.py index 1e713505..b4a7a4d7 100644 --- a/agate/table/exclude.py +++ b/agate/table/exclude.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils diff --git a/agate/table/find.py b/agate/table/find.py index d11ab1de..99dd373a 100644 --- a/agate/table/find.py +++ b/agate/table/find.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - def find(self, test): """ diff --git a/agate/table/group_by.py b/agate/table/group_by.py index b5c6a91c..b7bc405f 100644 --- a/agate/table/group_by.py +++ b/agate/table/group_by.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from collections import OrderedDict from agate.data_types import Text diff --git a/agate/table/homogenize.py b/agate/table/homogenize.py index 8a469f64..46c3fbe8 100644 --- a/agate/table/homogenize.py +++ b/agate/table/homogenize.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils from agate.rows import Row diff --git a/agate/table/join.py b/agate/table/join.py index 9b6ea37f..7ed2ea6c 100644 --- a/agate/table/join.py +++ b/agate/table/join.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils from agate.rows import Row diff --git a/agate/table/limit.py b/agate/table/limit.py index adc2988f..d164c183 100644 --- a/agate/table/limit.py +++ b/agate/table/limit.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - def limit(self, start_or_stop=None, stop=None, step=None): """ diff --git a/agate/table/line_chart.py b/agate/table/line_chart.py index 72c5f9df..3c40d3da 100644 --- a/agate/table/line_chart.py +++ b/agate/table/line_chart.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather diff --git a/agate/table/merge.py b/agate/table/merge.py index d47fbc72..0e50dd39 100644 --- a/agate/table/merge.py +++ b/agate/table/merge.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from collections import OrderedDict from agate.exceptions import DataTypeError diff --git a/agate/table/normalize.py b/agate/table/normalize.py index d4e4542f..0c0caa31 100644 --- a/agate/table/normalize.py +++ b/agate/table/normalize.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils from agate.rows import Row from agate.type_tester import TypeTester diff --git a/agate/table/order_by.py b/agate/table/order_by.py index 8ceff4df..7bfdf0bd 100644 --- a/agate/table/order_by.py +++ b/agate/table/order_by.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils diff --git a/agate/table/pivot.py b/agate/table/pivot.py index ae6cfe42..126805da 100644 --- a/agate/table/pivot.py +++ b/agate/table/pivot.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils from agate.aggregations import Count diff --git a/agate/table/print_bars.py b/agate/table/print_bars.py index 25dc3a2d..0f0cd589 100644 --- a/agate/table/print_bars.py +++ b/agate/table/print_bars.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import sys from collections import OrderedDict from decimal import Decimal diff --git a/agate/table/print_html.py b/agate/table/print_html.py index dd5cca86..610516a2 100644 --- a/agate/table/print_html.py +++ b/agate/table/print_html.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import math import sys diff --git a/agate/table/print_structure.py b/agate/table/print_structure.py index d099aea8..1c94d1a1 100644 --- a/agate/table/print_structure.py +++ b/agate/table/print_structure.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import sys from agate.data_types import Text diff --git a/agate/table/print_table.py b/agate/table/print_table.py index 6d502f5a..3490b1e0 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import math import sys diff --git a/agate/table/rename.py b/agate/table/rename.py index 14069ce3..b245d503 100644 --- a/agate/table/rename.py +++ b/agate/table/rename.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils diff --git a/agate/table/scatterplot.py b/agate/table/scatterplot.py index 671c9723..141c3aea 100644 --- a/agate/table/scatterplot.py +++ b/agate/table/scatterplot.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather diff --git a/agate/table/select.py b/agate/table/select.py index a1cd1ff9..8c999366 100644 --- a/agate/table/select.py +++ b/agate/table/select.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate import utils from agate.rows import Row diff --git a/agate/table/to_csv.py b/agate/table/to_csv.py index 9890fcac..e667ec48 100644 --- a/agate/table/to_csv.py +++ b/agate/table/to_csv.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import os diff --git a/agate/table/to_json.py b/agate/table/to_json.py index 7686a419..51afdc87 100644 --- a/agate/table/to_json.py +++ b/agate/table/to_json.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import json import os from collections import OrderedDict diff --git a/agate/table/where.py b/agate/table/where.py index cea3e36e..7127cb97 100644 --- a/agate/table/where.py +++ b/agate/table/where.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - def where(self, test): """ diff --git a/agate/tableset/aggregate.py b/agate/tableset/aggregate.py index e6aff37b..b6af1a08 100644 --- a/agate/tableset/aggregate.py +++ b/agate/tableset/aggregate.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate.table import Table diff --git a/agate/tableset/bar_chart.py b/agate/tableset/bar_chart.py index 26041987..6cf886ce 100644 --- a/agate/tableset/bar_chart.py +++ b/agate/tableset/bar_chart.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather diff --git a/agate/tableset/column_chart.py b/agate/tableset/column_chart.py index 7116c488..68342f8a 100644 --- a/agate/tableset/column_chart.py +++ b/agate/tableset/column_chart.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather diff --git a/agate/tableset/having.py b/agate/tableset/having.py index a3667351..6f6b466c 100644 --- a/agate/tableset/having.py +++ b/agate/tableset/having.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - def having(self, aggregations, test): """ diff --git a/agate/tableset/line_chart.py b/agate/tableset/line_chart.py index e28aefb7..51a569d7 100644 --- a/agate/tableset/line_chart.py +++ b/agate/tableset/line_chart.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather diff --git a/agate/tableset/merge.py b/agate/tableset/merge.py index a469fa64..2c83c7ad 100644 --- a/agate/tableset/merge.py +++ b/agate/tableset/merge.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - from agate.rows import Row from agate.table import Table diff --git a/agate/tableset/scatterplot.py b/agate/tableset/scatterplot.py index 391280b4..dace13d5 100644 --- a/agate/tableset/scatterplot.py +++ b/agate/tableset/scatterplot.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# pylint: disable=W0212 - import leather From 4dc005e355a155880234081eab6f94c38416054f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:42:06 -0400 Subject: [PATCH 139/163] test: Remove shebang --- agate/aggregations/sum.py | 1 - benchmarks/test_joins.py | 2 -- tests/test_computations.py | 2 -- 3 files changed, 5 deletions(-) diff --git a/agate/aggregations/sum.py b/agate/aggregations/sum.py index 0d45342a..7218317e 100644 --- a/agate/aggregations/sum.py +++ b/agate/aggregations/sum.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import datetime from agate.aggregations.base import Aggregation diff --git a/benchmarks/test_joins.py b/benchmarks/test_joins.py index 0c7ad9bb..c9e1432b 100644 --- a/benchmarks/test_joins.py +++ b/benchmarks/test_joins.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import unittest from random import shuffle from timeit import Timer diff --git a/tests/test_computations.py b/tests/test_computations.py index c7854ff6..8821f359 100644 --- a/tests/test_computations.py +++ b/tests/test_computations.py @@ -1,5 +1,3 @@ -#!/usr/bin/env Python - import datetime import unittest import warnings From 0857fdbc8cdade888d23e6a51878032c13e0c21a Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:51:51 -0400 Subject: [PATCH 140/163] test: Remove noqa --- tests/test_table/test_join.py | 2 +- tests/test_table/test_order_py.py | 3 +-- tests/test_table/test_print_bars.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_table/test_join.py b/tests/test_table/test_join.py index 1dd87e53..2d6d7878 100644 --- a/tests/test_table/test_join.py +++ b/tests/test_table/test_join.py @@ -215,7 +215,7 @@ def test_join_require_match(self): with self.assertRaises(ValueError): self.left.join(self.right, 'one', 'five', require_match=True) - new_table = self.left.join(self.right, 'one', 'four', require_match=True) # noqa + self.left.join(self.right, 'one', 'four', require_match=True) def test_join_columns_kwarg(self): new_table = self.left.join(self.right, 'one', 'four', columns=['six']) diff --git a/tests/test_table/test_order_py.py b/tests/test_table/test_order_py.py index e7f4e0a6..c6bbaa7d 100644 --- a/tests/test_table/test_order_py.py +++ b/tests/test_table/test_order_py.py @@ -143,5 +143,4 @@ def test_order_by_with_row_names(self): def test_order_by_empty_table(self): table = Table([], self.column_names) - - new_table = table.order_by('three') # noqa + table.order_by('three') diff --git a/tests/test_table/test_print_bars.py b/tests/test_table/test_print_bars.py index ea149b73..50ab0dfa 100644 --- a/tests/test_table/test_print_bars.py +++ b/tests/test_table/test_print_bars.py @@ -32,7 +32,7 @@ def test_print_bars(self): output = StringIO() table.print_bars('three', 'one', output=output) - lines = output.getvalue().split('\n') # noqa + output.getvalue().split('\n') def test_print_bars_width(self): table = Table(self.rows, self.column_names, self.column_types) From 91768084c7aeac6ed16a214dfcf2ef4423b1c06a Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:18:28 -0400 Subject: [PATCH 141/163] docs: Use autodoc_default_options --- agate/data_types/date.py | 3 +-- agate/data_types/date_time.py | 3 +-- docs/conf.py | 5 ++++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/agate/data_types/date.py b/agate/data_types/date.py index 4476bbd7..18d6e1e0 100644 --- a/agate/data_types/date.py +++ b/agate/data_types/date.py @@ -56,8 +56,7 @@ def cast(self, d): If both `date_format` and `locale` have been specified in the `agate.Date` instance, the `cast()` function is not thread-safe. - :returns: - :class:`datetime.date` or :code:`None`. + :returns: :class:`datetime.date` or :code:`None`. """ if type(d) is date or d is None: return d diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index a4e8a661..21fb21eb 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -63,8 +63,7 @@ def cast(self, d): If both `date_format` and `locale` have been specified in the `agate.DateTime` instance, the `cast()` function is not thread-safe. - :returns: - :class:`datetime.datetime` or :code:`None`. + :returns: :class:`datetime.datetime` or :code:`None`. """ if isinstance(d, datetime.datetime) or d is None: return d diff --git a/docs/conf.py b/docs/conf.py index c50a5a3a..fa64aab8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,7 +34,10 @@ htmlhelp_basename = 'agatedoc' -autodoc_default_flags = ['members', 'show-inheritance'] +autodoc_default_options = { + 'members': None, + 'show-inheritance': True, +} intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), From 66222ec0320ff6a4e652fef067e9fa7da6b704ee Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:40:35 -0400 Subject: [PATCH 142/163] fix: Allow consecutive calls to Table.group_by, closes #765 --- CHANGELOG.rst | 5 +++++ agate/tableset/__init__.py | 7 ++++++- docs/conf.py | 1 + tests/test_table/test_group_by.py | 5 +++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6d1ad98a..7827bf62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +1.7.2 +----- + +* Fix consecutive calls to :meth:`.Table.group_by`. (#765) + 1.7.1 - Jan 4, 2023 ------------------- diff --git a/agate/tableset/__init__.py b/agate/tableset/__init__.py index 4876d00b..7e70236f 100644 --- a/agate/tableset/__init__.py +++ b/agate/tableset/__init__.py @@ -148,7 +148,12 @@ def _proxy(self, method_name, *args, **kwargs): tables = [] for key, table in self.items(): - tables.append(getattr(table, method_name)(*args, **kwargs)) + result = getattr(table, method_name)(*args, **kwargs) + if isinstance(result, TableSet): + for table in result.values(): + tables.append(table) + else: + tables.append(result) return self._fork( tables, diff --git a/docs/conf.py b/docs/conf.py index fa64aab8..4c341464 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,6 +36,7 @@ autodoc_default_options = { 'members': None, + 'member-order': 'bysource', 'show-inheritance': True, } diff --git a/tests/test_table/test_group_by.py b/tests/test_table/test_group_by.py index cb51ffd4..6178a3a8 100644 --- a/tests/test_table/test_group_by.py +++ b/tests/test_table/test_group_by.py @@ -108,3 +108,8 @@ def test_group_by_bad_column(self): with self.assertRaises(KeyError): table.group_by('bad') + + def test_group_by_twice(self): + table = Table(self.rows, self.column_names, self.column_types) + + repr(table.group_by('one').group_by('two')) From 469ee27b464974960df8b76cfaf8ac56fbfc2e92 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 23:07:38 -0400 Subject: [PATCH 143/163] docs: Update CHANGELOG and AUTHORS --- AUTHORS.rst | 1 + CHANGELOG.rst | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 7f9b8275..a6f367e4 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -45,3 +45,4 @@ agate is made by a community. The following individuals have contributed code, d * `brian-from-quantrocket `_ * `mathdesc `_ * `Tim Gates `_ +* `castorf `_ diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7827bf62..169c3290 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,8 @@ -1.7.2 +1.8.0 ----- -* Fix consecutive calls to :meth:`.Table.group_by`. (#765) +* feat: Lowercase the ``null_values`` provided to individual data types, since all comparisons to ``null_values`` are case-insensitive. (#770) +* fix: Allow consecutive calls to :meth:`.Table.group_by`. (#765) 1.7.1 - Jan 4, 2023 ------------------- From 6ef58845b0b4c2a71b33027770940ce5786106b2 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 23:29:39 -0400 Subject: [PATCH 144/163] feat: Mean() works with TimeDelta(), #761 --- CHANGELOG.rst | 1 + agate/aggregations/mean.py | 11 +++++++---- tests/test_aggregations.py | 29 ++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 169c3290..53574945 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,7 @@ ----- * feat: Lowercase the ``null_values`` provided to individual data types, since all comparisons to ``null_values`` are case-insensitive. (#770) +* feat: :class:`.Mean` works with :class:`.TimeDelta`. (#761) * fix: Allow consecutive calls to :meth:`.Table.group_by`. (#765) 1.7.1 - Jan 4, 2023 diff --git a/agate/aggregations/mean.py b/agate/aggregations/mean.py index 107466f3..9fe27dc9 100644 --- a/agate/aggregations/mean.py +++ b/agate/aggregations/mean.py @@ -1,7 +1,7 @@ from agate.aggregations.base import Aggregation from agate.aggregations.has_nulls import HasNulls from agate.aggregations.sum import Sum -from agate.data_types import Number +from agate.data_types import Number, TimeDelta from agate.exceptions import DataTypeError from agate.warns import warn_null_calculation @@ -18,13 +18,16 @@ def __init__(self, column_name): self._sum = Sum(column_name) def get_aggregate_data_type(self, table): - return Number() + column = table.columns[self._column_name] + + if isinstance(column.data_type, (Number, TimeDelta)): + return column.data_type def validate(self, table): column = table.columns[self._column_name] - if not isinstance(column.data_type, Number): - raise DataTypeError('Mean can only be applied to columns containing Number data.') + if not isinstance(column.data_type, (Number, TimeDelta)): + raise DataTypeError('Mean can only be applied to columns containing Number or TimeDelta data.') has_nulls = HasNulls(self._column_name).run(table) diff --git a/tests/test_aggregations.py b/tests/test_aggregations.py index c7fd352a..c8dba876 100644 --- a/tests/test_aggregations.py +++ b/tests/test_aggregations.py @@ -184,11 +184,13 @@ def setUp(self): self.table = Table(self.rows, ['test', 'null'], [DateTime(), DateTime()]) self.time_delta_rows = [ - [datetime.timedelta(seconds=10), None], - [datetime.timedelta(seconds=20), None], + [datetime.timedelta(seconds=10), datetime.timedelta(seconds=15), None], + [datetime.timedelta(seconds=20), None, None], ] - self.time_delta_table = Table(self.time_delta_rows, ['test', 'null'], [TimeDelta(), TimeDelta()]) + self.time_delta_table = Table( + self.time_delta_rows, ['test', 'mixed', 'null'], [TimeDelta(), TimeDelta(), TimeDelta()] + ) def test_min(self): self.assertIsInstance(Min('test').get_aggregate_data_type(self.table), DateTime) @@ -216,6 +218,27 @@ def test_max_time_delta(self): Max('test').validate(self.time_delta_table) self.assertEqual(Max('test').run(self.time_delta_table), datetime.timedelta(0, 20)) + def test_mean(self): + with self.assertWarns(NullCalculationWarning): + Mean('mixed').validate(self.time_delta_table) + + Mean('test').validate(self.time_delta_table) + + self.assertEqual(Mean('test').run(self.time_delta_table), datetime.timedelta(seconds=15)) + + def test_mean_all_nulls(self): + self.assertIsNone(Mean('null').run(self.time_delta_table)) + + def test_mean_with_nulls(self): + warnings.simplefilter('ignore') + + try: + Mean('mixed').validate(self.time_delta_table) + finally: + warnings.resetwarnings() + + self.assertAlmostEqual(Mean('mixed').run(self.time_delta_table), datetime.timedelta(seconds=15)) + def test_sum(self): self.assertIsInstance(Sum('test').get_aggregate_data_type(self.time_delta_table), TimeDelta) Sum('test').validate(self.time_delta_table) From aa3e1cc1cc71c42c660c3418c50fd7d89d89757f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 3 Oct 2023 23:58:28 -0400 Subject: [PATCH 145/163] docs: Fix unreleased version number --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53574945..8500991d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,5 @@ -1.8.0 ------ +Unreleased +---------- * feat: Lowercase the ``null_values`` provided to individual data types, since all comparisons to ``null_values`` are case-insensitive. (#770) * feat: :class:`.Mean` works with :class:`.TimeDelta`. (#761) From 7328f45a163f6032ae26126777fdd6b0ecdf61a8 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 4 Oct 2023 00:42:45 -0400 Subject: [PATCH 146/163] ci: Report coverage to Coveralls --- .github/workflows/ci.yml | 3 +++ README.rst | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 472e7576..50915250 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,3 +35,6 @@ jobs: PYTHONUTF8: 1 run: pytest --cov agate - run: python charts.py + - env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: coveralls --service=github diff --git a/README.rst b/README.rst index 558d5d2f..321e333f 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,10 @@ :target: https://github.com/wireservice/agate/actions :alt: Build status +.. image:: https://coveralls.io/repos/wireservice/agate/badge.svg?branch=master + :target: https://coveralls.io/r/wireservice/agate + :alt: Coverage status + .. image:: https://img.shields.io/pypi/dm/agate.svg :target: https://pypi.python.org/pypi/agate :alt: PyPI downloads From 5021bd5dc31b0df62fac6796a7b102e518c00f13 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 4 Oct 2023 00:51:11 -0400 Subject: [PATCH 147/163] ci: Install coveralls package --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50915250..57671348 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: setup.py - - run: pip install .[test] + - run: pip install .[test] coveralls - env: LANG: en_US.UTF-8 PYTHONIOENCODING: utf-8 From 5998a83af9ab45052e7483cbe07cb644f4e15c6b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 4 Oct 2023 01:07:58 -0400 Subject: [PATCH 148/163] Revert "fix: Allow consecutive calls to Table.group_by, closes #765" --- CHANGELOG.rst | 1 - agate/tableset/__init__.py | 7 +------ tests/test_table/test_group_by.py | 5 ----- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8500991d..03109549 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,6 @@ Unreleased * feat: Lowercase the ``null_values`` provided to individual data types, since all comparisons to ``null_values`` are case-insensitive. (#770) * feat: :class:`.Mean` works with :class:`.TimeDelta`. (#761) -* fix: Allow consecutive calls to :meth:`.Table.group_by`. (#765) 1.7.1 - Jan 4, 2023 ------------------- diff --git a/agate/tableset/__init__.py b/agate/tableset/__init__.py index 7e70236f..4876d00b 100644 --- a/agate/tableset/__init__.py +++ b/agate/tableset/__init__.py @@ -148,12 +148,7 @@ def _proxy(self, method_name, *args, **kwargs): tables = [] for key, table in self.items(): - result = getattr(table, method_name)(*args, **kwargs) - if isinstance(result, TableSet): - for table in result.values(): - tables.append(table) - else: - tables.append(result) + tables.append(getattr(table, method_name)(*args, **kwargs)) return self._fork( tables, diff --git a/tests/test_table/test_group_by.py b/tests/test_table/test_group_by.py index 6178a3a8..cb51ffd4 100644 --- a/tests/test_table/test_group_by.py +++ b/tests/test_table/test_group_by.py @@ -108,8 +108,3 @@ def test_group_by_bad_column(self): with self.assertRaises(KeyError): table.group_by('bad') - - def test_group_by_twice(self): - table = Table(self.rows, self.column_names, self.column_types) - - repr(table.group_by('one').group_by('two')) From c88192a5ac94bbeb8a5c967f6581bcf2c36e1d8f Mon Sep 17 00:00:00 2001 From: Julien Enselme Date: Wed, 1 Mar 2023 08:53:37 +0100 Subject: [PATCH 149/163] Remove dependency on pytz Starting with Python 3.9 a zoneinfo package is available in the standard library for this feature. There is therefore no need for this dependency and a compatibility layer is available at https://pypi.org/project/backports.zoneinfo/ for older Python versions. --- CHANGELOG.rst | 1 + agate/data_types/date_time.py | 3 +-- docs/cookbook/datetime.rst | 18 +++++++++++++----- setup.py | 3 ++- tests/test_data_types.py | 15 ++++++++++----- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 03109549..fb7f31a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,7 @@ Unreleased * feat: Lowercase the ``null_values`` provided to individual data types, since all comparisons to ``null_values`` are case-insensitive. (#770) * feat: :class:`.Mean` works with :class:`.TimeDelta`. (#761) +* Switch from ``pytz`` to ``ZoneInfo``. 1.7.1 - Jan 4, 2023 ------------------- diff --git a/agate/data_types/date_time.py b/agate/data_types/date_time.py index 21fb21eb..71fad4fd 100644 --- a/agate/data_types/date_time.py +++ b/agate/data_types/date_time.py @@ -16,8 +16,7 @@ class DateTime(DataType): A formatting string for :meth:`datetime.datetime.strptime` to use instead of using regex-based parsing. :param timezone: - A `pytz `_ timezone to apply to each - parsed date. + A ``ZoneInfo`` timezone to apply to each parsed date. :param locale: A locale specification such as :code:`en_US` or :code:`de_DE` to use for parsing formatted datetimes. diff --git a/docs/cookbook/datetime.rst b/docs/cookbook/datetime.rst index 78ffcf0c..5f9f781a 100644 --- a/docs/cookbook/datetime.rst +++ b/docs/cookbook/datetime.rst @@ -34,9 +34,13 @@ The second way is to specify a timezone as an argument to the type constructor: .. code-block:: python - import pytz + try: + from zoneinfo import ZoneInfo + except ImportError: + # Fallback for Python < 3.9 + from backports.zoneinfo import ZoneInfo - eastern = pytz.timezone('US/Eastern') + eastern = ZoneInfo('US/Eastern') datetime_type = agate.DateTime(timezone=eastern) In this case all timezones that are processed will be set to have the Eastern timezone. Note, the timezone will be **set**, not converted. You cannot use this method to convert your timezones from UTC to another timezone. To do that see :ref:`convert_timezones`. @@ -60,9 +64,13 @@ If you load data from a spreadsheet in one timezone and you need to convert it t .. code-block:: python - import pytz + try: + from zoneinfo import ZoneInfo + except ImportError: + # Fallback for Python < 3.9 + from backports.zoneinfo import ZoneInfo - us_eastern = pytz.timezone('US/Eastern') + us_eastern = ZoneInfo('US/Eastern') datetime_type = agate.DateTime(timezone=us_eastern) column_names = ['what', 'when'] @@ -70,7 +78,7 @@ If you load data from a spreadsheet in one timezone and you need to convert it t table = agate.Table.from_csv('events.csv', columns) - rome = timezone('Europe/Rome') + rome = ZoneInfo('Europe/Rome') timezone_shifter = agate.Formula(lambda r: r['when'].astimezone(rome)) table = agate.Table.compute([ diff --git a/setup.py b/setup.py index 3aa57f42..ba53e4c8 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ 'parsedatetime>=2.1,!=2.5', 'python-slugify>=1.2.1', 'pytimeparse>=1.1.5', + 'tzdata>=2023.3;platform_system=="Windows"', ], extras_require={ 'test': [ @@ -54,7 +55,7 @@ 'PyICU>=2.4.2;sys_platform=="linux"', 'pytest', 'pytest-cov', - 'pytz>=2015.4', + 'backports.zoneinfo;python_version<"3.9"', ], } ) diff --git a/tests/test_data_types.py b/tests/test_data_types.py index 842b9188..4f2bac53 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -4,7 +4,12 @@ from decimal import Decimal import parsedatetime -import pytz + +try: + from zoneinfo import ZoneInfo +except ImportError: + # Fallback for Python < 3.9 + from backports.zoneinfo import ZoneInfo from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.exceptions import CastError @@ -352,16 +357,16 @@ def test_cast_parser(self): )) def test_cast_parser_timezone(self): - tzinfo = pytz.timezone('US/Pacific') + tzinfo = ZoneInfo('US/Pacific') datetime_type = DateTime(timezone=tzinfo) values = ('3/1/1994 12:30 PM', '2/17/2011 06:30', None, 'January 5th, 1984 22:37', 'n/a') casted = tuple(datetime_type.cast(v) for v in values) self.assertSequenceEqual(casted, ( - tzinfo.localize(datetime.datetime(1994, 3, 1, 12, 30, 0, 0)), - tzinfo.localize(datetime.datetime(2011, 2, 17, 6, 30, 0, 0)), + datetime.datetime(1994, 3, 1, 12, 30, 0, 0, tzinfo=tzinfo), + datetime.datetime(2011, 2, 17, 6, 30, 0, 0, tzinfo=tzinfo), None, - tzinfo.localize(datetime.datetime(1984, 1, 5, 22, 37, 0, 0)), + datetime.datetime(1984, 1, 5, 22, 37, 0, 0, tzinfo=tzinfo), None )) From 4c327657faf01e36706c5c6e072ac29c275a3326 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:30:03 -0400 Subject: [PATCH 150/163] ci: Test on Python 3.12 --- .github/workflows/ci.yml | 7 +------ AUTHORS.rst | 2 ++ CHANGELOG.rst | 2 ++ setup.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57671348..b68f37b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,12 +7,7 @@ jobs: strategy: matrix: os: [macos-latest, windows-latest, ubuntu-latest] - python-version: [3.7, 3.8, 3.9, '3.10', '3.11', pypy-3.7] - exclude: - - os: windows-latest - python-version: 3.7 - - os: windows-latest - python-version: pypy-3.7 + python-version: [3.8, 3.9, '3.10', '3.11', '3.12', pypy-3.9] steps: - if: matrix.os == 'ubuntu-latest' name: Install UTF-8 locales and lxml requirements diff --git a/AUTHORS.rst b/AUTHORS.rst index a6f367e4..ca5e1b32 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -46,3 +46,5 @@ agate is made by a community. The following individuals have contributed code, d * `mathdesc `_ * `Tim Gates `_ * `castorf `_ +* `Julien Enselme `__ + diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fb7f31a8..97f5e8cb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,8 @@ Unreleased * feat: Lowercase the ``null_values`` provided to individual data types, since all comparisons to ``null_values`` are case-insensitive. (#770) * feat: :class:`.Mean` works with :class:`.TimeDelta`. (#761) * Switch from ``pytz`` to ``ZoneInfo``. +* Add Python 3.12 support. +* Drop Python 3.7 support (end-of-life was June 27, 2023). 1.7.1 - Jan 4, 2023 ------------------- diff --git a/setup.py b/setup.py index ba53e4c8..c49bf225 100644 --- a/setup.py +++ b/setup.py @@ -25,11 +25,11 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Scientific/Engineering :: Information Analysis', From 9614bb24673a9cd12444514f4c9544dbb2a8996f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:42:54 -0400 Subject: [PATCH 151/163] docs(changelog): Reduce duplication --- CHANGELOG.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 97f5e8cb..0ebef71b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,10 +15,8 @@ Unreleased 1.7.0 - Jan 3, 2023 ------------------- -* Add Python 3.11 support. -* Add Python 3.10 support. -* Drop Python 3.6 support (end-of-life was December 23, 2021). -* Drop Python 2.7 support (end-of-life was January 1, 2020). +* Add Python 3.10 and 3.11 support. +* Drop support for Python 2.7 (EOL 2020-01-01), 3.6 (2021-12-23). 1.6.3 - July 15, 2021 --------------------- @@ -39,8 +37,7 @@ Unreleased * fix: Aggregations return ``None`` if all values are ``None``, instead of raising an error. Note that ``Sum``, ``MaxLength`` and ``MaxPrecision`` continue to return ``0`` if all values are ``None``. (#706) * fix: Ensure files are closed when errors occur. (#734) * build: Make PyICU an optional dependency. -* Drop Python 3.5 support (end-of-life was September 13, 2020). -* Drop Python 3.4 support (end-of-life was March 18, 2019). +* Drop support for Python 3.4 (2019-03-18), 3.5 (2020-09-13). 1.6.2 - March 10, 2021 ---------------------- From 7c825992e24fe05bd7f2277968711a4824d999ae Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:49:19 -0400 Subject: [PATCH 152/163] build: Iterate the version number --- CHANGELOG.rst | 12 ++++++------ docs/conf.py | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0ebef71b..3b80f421 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,5 @@ -Unreleased ----------- +1.8.0 - October 10, 2023 +------------------------ * feat: Lowercase the ``null_values`` provided to individual data types, since all comparisons to ``null_values`` are case-insensitive. (#770) * feat: :class:`.Mean` works with :class:`.TimeDelta`. (#761) @@ -7,13 +7,13 @@ Unreleased * Add Python 3.12 support. * Drop Python 3.7 support (end-of-life was June 27, 2023). -1.7.1 - Jan 4, 2023 -------------------- +1.7.1 - January 4, 2023 +----------------------- * Allow parsedatetime 2.6. -1.7.0 - Jan 3, 2023 -------------------- +1.7.0 - January 3, 2023 +----------------------- * Add Python 3.10 and 3.11 support. * Drop support for Python 2.7 (EOL 2020-01-01), 3.6 (2021-12-23). diff --git a/docs/conf.py b/docs/conf.py index 4c341464..fd23c0bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ project = 'agate' copyright = '2017, Christopher Groskopf' -version = '1.7.1' +version = '1.8.0' release = version # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index c49bf225..8d77a5f8 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='agate', - version='1.7.1', + version='1.8.0', description='A data analysis library that is optimized for humans instead of machines.', long_description=long_description, long_description_content_type='text/x-rst', From 933119228b2696747979b779938a117d23631adf Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 10 Oct 2023 13:06:51 -0400 Subject: [PATCH 153/163] chore: Run pyupgrade --py38-plus **/*.py --- agate/__init__.py | 1 - agate/aggregations/base.py | 1 - agate/computations/base.py | 1 - agate/computations/percent.py | 1 - agate/data_types/base.py | 1 - agate/data_types/text.py | 1 - agate/table/find.py | 1 - agate/table/limit.py | 1 - agate/table/pivot.py | 1 - agate/table/where.py | 1 - agate/tableset/having.py | 1 - agate/tableset/proxy_methods.py | 1 - tests/test_from_json.py | 1 - tests/test_table/test_compute.py | 1 - 14 files changed, 14 deletions(-) diff --git a/agate/__init__.py b/agate/__init__.py index 339fe694..77a01443 100644 --- a/agate/__init__.py +++ b/agate/__init__.py @@ -1,4 +1,3 @@ - import agate.csv_py3 as csv from agate.aggregations import * from agate.columns import Column diff --git a/agate/aggregations/base.py b/agate/aggregations/base.py index aa7b48d8..307920f3 100644 --- a/agate/aggregations/base.py +++ b/agate/aggregations/base.py @@ -1,4 +1,3 @@ - from agate.exceptions import UnsupportedAggregationError diff --git a/agate/computations/base.py b/agate/computations/base.py index 69577aed..59b27dca 100644 --- a/agate/computations/base.py +++ b/agate/computations/base.py @@ -1,4 +1,3 @@ - class Computation: # pragma: no cover """ Computations produce a new column by performing a calculation on each row. diff --git a/agate/computations/percent.py b/agate/computations/percent.py index da3a20ff..3fb440e0 100644 --- a/agate/computations/percent.py +++ b/agate/computations/percent.py @@ -1,4 +1,3 @@ - from agate.aggregations.has_nulls import HasNulls from agate.aggregations.sum import Sum from agate.computations.base import Computation diff --git a/agate/data_types/base.py b/agate/data_types/base.py index aaa4b5e2..951ab072 100644 --- a/agate/data_types/base.py +++ b/agate/data_types/base.py @@ -1,4 +1,3 @@ - from agate.exceptions import CastError #: Default values which will be automatically cast to :code:`None` diff --git a/agate/data_types/text.py b/agate/data_types/text.py index 178e43cc..263d2157 100644 --- a/agate/data_types/text.py +++ b/agate/data_types/text.py @@ -1,4 +1,3 @@ - from agate.data_types.base import DataType diff --git a/agate/table/find.py b/agate/table/find.py index 99dd373a..d13d7cdd 100644 --- a/agate/table/find.py +++ b/agate/table/find.py @@ -1,4 +1,3 @@ - def find(self, test): """ Find the first row that passes a test. diff --git a/agate/table/limit.py b/agate/table/limit.py index d164c183..701e6693 100644 --- a/agate/table/limit.py +++ b/agate/table/limit.py @@ -1,4 +1,3 @@ - def limit(self, start_or_stop=None, stop=None, step=None): """ Create a new table with fewer rows. diff --git a/agate/table/pivot.py b/agate/table/pivot.py index 126805da..f74d848c 100644 --- a/agate/table/pivot.py +++ b/agate/table/pivot.py @@ -1,4 +1,3 @@ - from agate import utils from agate.aggregations import Count diff --git a/agate/table/where.py b/agate/table/where.py index 7127cb97..90259771 100644 --- a/agate/table/where.py +++ b/agate/table/where.py @@ -1,4 +1,3 @@ - def where(self, test): """ Create a new :class:`.Table` with only those rows that pass a test. diff --git a/agate/tableset/having.py b/agate/tableset/having.py index 6f6b466c..bb0f46ac 100644 --- a/agate/tableset/having.py +++ b/agate/tableset/having.py @@ -1,4 +1,3 @@ - def having(self, aggregations, test): """ Create a new :class:`.TableSet` with only those tables that pass a test. diff --git a/agate/tableset/proxy_methods.py b/agate/tableset/proxy_methods.py index 4b613c50..e7ac2b75 100644 --- a/agate/tableset/proxy_methods.py +++ b/agate/tableset/proxy_methods.py @@ -1,4 +1,3 @@ - def bins(self, *args, **kwargs): """ Calls :meth:`.Table.bins` on each table in the TableSet. diff --git a/tests/test_from_json.py b/tests/test_from_json.py index 2dcec013..e880e153 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -1,4 +1,3 @@ - from agate import Table from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta from agate.testcase import AgateTestCase diff --git a/tests/test_table/test_compute.py b/tests/test_table/test_compute.py index 530dc12b..976ee2dd 100644 --- a/tests/test_table/test_compute.py +++ b/tests/test_table/test_compute.py @@ -1,4 +1,3 @@ - from agate import Table from agate.computations import Formula from agate.data_types import Number, Text From 71211ad2e767a48fbc5208876f54263e9506a8c2 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 17 Oct 2023 21:47:10 -0400 Subject: [PATCH 154/163] docs: Document that "/" in JSON keys can cause collisions, closes #767 --- agate/table/from_object.py | 19 +++++++++++++++++++ examples/test_from_json_ambiguous.json | 8 ++++++++ tests/test_from_json.py | 8 ++++++++ 3 files changed, 35 insertions(+) create mode 100644 examples/test_from_json_ambiguous.json diff --git a/agate/table/from_object.py b/agate/table/from_object.py index f5357f05..f114e918 100644 --- a/agate/table/from_object.py +++ b/agate/table/from_object.py @@ -40,6 +40,25 @@ def from_object(cls, obj, row_names=None, column_types=None): Not all rows are required to have the same keys. Missing elements will be filled in with null values. + Keys containing a slash (``/``) can collide with other keys. For example: + + .. code-block:: python + + { + 'a/b': 2, + 'a': { + 'b': False + } + } + + Would generate: + + .. code-block:: python + + { + 'a/b': false + } + :param obj: Filepath or file-like object from which to read JSON data. :param row_names: diff --git a/examples/test_from_json_ambiguous.json b/examples/test_from_json_ambiguous.json new file mode 100644 index 00000000..5435946e --- /dev/null +++ b/examples/test_from_json_ambiguous.json @@ -0,0 +1,8 @@ +[ + { + "a/b": 2, + "a": { + "b": false + } + } +] diff --git a/tests/test_from_json.py b/tests/test_from_json.py index e880e153..63ba0602 100644 --- a/tests/test_from_json.py +++ b/tests/test_from_json.py @@ -1,5 +1,6 @@ from agate import Table from agate.data_types import Boolean, Date, DateTime, Number, Text, TimeDelta +from agate.rows import Row from agate.testcase import AgateTestCase from agate.type_tester import TypeTester @@ -85,3 +86,10 @@ def test_from_json_no_type_tester(self): def test_from_json_error_newline_key(self): with self.assertRaises(ValueError): Table.from_json('examples/test.json', newline=True, key='test') + + def test_from_json_ambiguous(self): + table = Table.from_json('examples/test_from_json_ambiguous.json') + + self.assertColumnNames(table, ('a/b',)) + self.assertColumnTypes(table, [Boolean]) + self.assertRows(table, [Row([False])]) From 11ea63e50709c6dc285769b9910a1d8293993746 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 17 Oct 2023 23:33:16 -0400 Subject: [PATCH 155/163] docs: Document that the lineterminator defaults to the newline character, closes #669 --- agate/table/to_csv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agate/table/to_csv.py b/agate/table/to_csv.py index e667ec48..4be7a96c 100644 --- a/agate/table/to_csv.py +++ b/agate/table/to_csv.py @@ -6,7 +6,9 @@ def to_csv(self, path, **kwargs): Write this table to a CSV. This method uses agate's builtin CSV writer, which supports unicode on both Python 2 and Python 3. - `kwargs` will be passed through to the CSV writer. + ``kwargs`` will be passed through to the CSV writer. + + The ``lineterminator`` defaults to the newline character (LF, ``\\n``). :param path: Filepath or file-like object to write to. From 01f094b2f22e31873120e5ed93e4ceb3fa576b99 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 17 Oct 2023 23:35:18 -0400 Subject: [PATCH 156/163] test: Skip failing test on GitHub Actions --- tests/test_py3.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_py3.py b/tests/test_py3.py index de24373c..e087cbc8 100644 --- a/tests/test_py3.py +++ b/tests/test_py3.py @@ -1,5 +1,7 @@ import csv import os +import platform +import sys import unittest from io import StringIO @@ -231,6 +233,10 @@ def test_writerows(self): class TestSniffer(unittest.TestCase): + @unittest.skipIf( + platform.system() == 'Darwin' and sys.version_info[:2] == (3, 10), + reason='The (macos-latest, 3.10) job fails on GitHub Actions' + ) def test_sniffer(self): with open('examples/test.csv', encoding='utf-8') as f: contents = f.read() From 9e79a33504bb97ab5c60a214ee6918f6ea8aa1b3 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 17 Oct 2023 23:51:44 -0400 Subject: [PATCH 157/163] feat: Add text_truncation_chars and number_truncation_chars configurations, #738 --- CHANGELOG.rst | 6 ++++++ agate/config.py | 8 ++++++++ agate/table/print_html.py | 4 +++- agate/table/print_table.py | 6 ++++-- agate/utils.py | 3 ++- docs/conf.py | 2 +- setup.py | 2 +- 7 files changed, 25 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b80f421..e69ba537 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,9 @@ +1.9.0 - October 17, 2023 +------------------------ + +* feat: Add a ``text_truncation_chars`` configuration for values that exceed ``max_column_width`` in :meth:`.Table.print_table` and :meth:`.Table.print_html`. +* feat: Add a ``number_truncation_chars`` configuration for values that exceed ``max_precision`` in :meth:`.Table.print_table` and :meth:`.Table.print_html`. + 1.8.0 - October 10, 2023 ------------------------ diff --git a/agate/config.py b/agate/config.py index b20b275e..f79ee875 100644 --- a/agate/config.py +++ b/agate/config.py @@ -26,6 +26,10 @@ +-------------------------+------------------------------------------+-----------------------------------------+ | ellipsis_chars | Characters to render for ellipsis | '...' | +-------------------------+------------------------------------------+-----------------------------------------+ +| text_truncation_chars | Characters for truncated text values | '...' | ++-------------------------+------------------------------------------+-----------------------------------------+ +| number_truncation_chars | Characters for truncated number values | '…' | ++-------------------------+------------------------------------------+-----------------------------------------+ """ @@ -50,6 +54,10 @@ 'tick_char': '+', #: Characters to render for ellipsis 'ellipsis_chars': '...', + #: Characters for truncated text values + 'text_truncation_chars': '...', + #: Characters for truncated number values + 'number_truncation_chars': '…', } diff --git a/agate/table/print_html.py b/agate/table/print_html.py index 610516a2..41c0837b 100644 --- a/agate/table/print_html.py +++ b/agate/table/print_html.py @@ -43,6 +43,8 @@ def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_w max_precision = float('inf') ellipsis = config.get_option('ellipsis_chars') + truncation = config.get_option('text_truncation_chars') + len_truncation = len(truncation) locale = locale or config.get_option('default_locale') rows_truncated = max_rows < len(self._rows) @@ -93,7 +95,7 @@ def print_html(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_w v = str(v) if max_column_width is not None and len(v) > max_column_width: - v = '%s...' % v[:max_column_width - 3] + v = '%s%s' % (v[:max_column_width - len_truncation], truncation) formatted_row.append(v) diff --git a/agate/table/print_table.py b/agate/table/print_table.py index 3490b1e0..d066488a 100644 --- a/agate/table/print_table.py +++ b/agate/table/print_table.py @@ -45,6 +45,8 @@ def print_table(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_ max_precision = float('inf') ellipsis = config.get_option('ellipsis_chars') + truncation = config.get_option('text_truncation_chars') + len_truncation = len(truncation) h_line = config.get_option('horizontal_line_char') v_line = config.get_option('vertical_line_char') locale = locale or config.get_option('default_locale') @@ -54,7 +56,7 @@ def print_table(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_ column_names = [] for column_name in self.column_names[:max_columns]: if max_column_width is not None and len(column_name) > max_column_width: - column_names.append('%s...' % column_name[:max_column_width - 3]) + column_names.append('%s%s' % (column_name[:max_column_width - len_truncation], truncation)) else: column_names.append(column_name) @@ -102,7 +104,7 @@ def print_table(self, max_rows=20, max_columns=6, output=sys.stdout, max_column_ v = str(v) if max_column_width is not None and len(v) > max_column_width: - v = '%s...' % v[:max_column_width - 3] + v = '%s%s' % (v[:max_column_width - len_truncation], truncation) if len(v) > widths[j]: widths[j] = len(v) diff --git a/agate/utils.py b/agate/utils.py index fc290c18..69bb01eb 100644 --- a/agate/utils.py +++ b/agate/utils.py @@ -12,6 +12,7 @@ from slugify import slugify as pslugify +from agate import config from agate.warns import warn_duplicate_column, warn_unnamed_column #: Sentinal for use when `None` is an valid argument value @@ -161,7 +162,7 @@ def make_number_formatter(decimal_places, add_ellipsis=False): Optionally add an ellipsis symbol at the end of a number """ fraction = '0' * decimal_places - ellipsis = '…' if add_ellipsis else '' + ellipsis = config.get_option('number_truncation_chars') if add_ellipsis else '' return ''.join(['#,##0.', fraction, ellipsis, ';-#,##0.', fraction, ellipsis]) diff --git a/docs/conf.py b/docs/conf.py index fd23c0bb..5907e574 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ project = 'agate' copyright = '2017, Christopher Groskopf' -version = '1.8.0' +version = '1.9.0' release = version # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 8d77a5f8..9b1c5383 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='agate', - version='1.8.0', + version='1.9.0', description='A data analysis library that is optimized for humans instead of machines.', long_description=long_description, long_description_content_type='text/x-rst', From b18899053a33e890f9ee9f9fa90e99dbdf03fb21 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 21 Dec 2023 15:02:21 -0500 Subject: [PATCH 158/163] Add Babel 2.14 support --- CHANGELOG.rst | 5 +++++ agate/data_types/number.py | 7 +++++-- docs/conf.py | 2 +- setup.py | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e69ba537..705cbe39 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +1.9.1 - December 21, 2023 +------------------------- + +* Add Babel 2.14 support. + 1.9.0 - October 17, 2023 ------------------------ diff --git a/agate/data_types/number.py b/agate/data_types/number.py index bc29fda6..b6dbee99 100644 --- a/agate/data_types/number.py +++ b/agate/data_types/number.py @@ -43,8 +43,11 @@ def __init__(self, locale='en_US', group_symbol=None, decimal_symbol=None, with warnings.catch_warnings(): warnings.simplefilter("ignore") - self.group_symbol = group_symbol or self.locale.number_symbols.get('group', ',') - self.decimal_symbol = decimal_symbol or self.locale.number_symbols.get('decimal', '.') + # Babel 2.14 support. + # https://babel.pocoo.org/en/latest/changelog.html#possibly-backwards-incompatible-changes + number_symbols = self.locale.number_symbols.get('latn', self.locale.number_symbols) + self.group_symbol = group_symbol or number_symbols.get('group', ',') + self.decimal_symbol = decimal_symbol or number_symbols.get('decimal', '.') def cast(self, d): """ diff --git a/docs/conf.py b/docs/conf.py index 5907e574..4b9a07bc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ project = 'agate' copyright = '2017, Christopher Groskopf' -version = '1.9.0' +version = '1.9.1' release = version # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 9b1c5383..49851d19 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='agate', - version='1.9.0', + version='1.9.1', description='A data analysis library that is optimized for humans instead of machines.', long_description=long_description, long_description_content_type='text/x-rst', From fc3dfb2021217a070e3263bb700c1b69bc8a2e90 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Thu, 21 Dec 2023 15:10:06 -0500 Subject: [PATCH 159/163] ci: Remove accidental copy-paste --- .github/workflows/lint.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 28c49bb4..72f1ccd2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,7 +1,5 @@ name: Lint on: [push, pull_request] -env: - BASEDIR: https://raw.githubusercontent.com/open-contracting/standard-maintenance-scripts/main jobs: build: if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository From e90edce6f361e3ca4a0397793e5b29bb40337d80 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:38:35 -0500 Subject: [PATCH 160/163] ci: Upgrade actions. Add dependabot.yml. --- .github/workflows/ci.yml | 2 +- .github/workflows/dependabot.yml | 6 ++++++ .github/workflows/lint.yml | 2 +- .github/workflows/pypi.yml | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/dependabot.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b68f37b3..20cc72b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: sudo locale-gen en_US.UTF-8 sudo locale-gen ko_KR.UTF-8 sudo update-locale - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml new file mode 100644 index 00000000..12301490 --- /dev/null +++ b/.github/workflows/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 72f1ccd2..27daff70 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,7 +5,7 @@ jobs: if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3.10' diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 2654ebbc..2344e1e6 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -4,7 +4,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3.10' From ac2449993289d6074839a17ad1bc9853ad1b860d Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:39:44 -0500 Subject: [PATCH 161/163] ci: Upgrade actions --- .github/workflows/ci.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pypi.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20cc72b4..2ee35ea3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: sudo locale-gen ko_KR.UTF-8 sudo update-locale - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 27daff70..e4a09a27 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' cache: pip diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 2344e1e6..1e9ee54d 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - run: pip install --upgrade build From 7a9b55408c69e07c9b40bd4794b27f9a9918dd05 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:55:39 -0500 Subject: [PATCH 162/163] ci: Move dependabot.yml --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..12301490 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" From 80106fdb851204071490a036471418eb690ae7c3 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:56:51 -0500 Subject: [PATCH 163/163] ci: Delete workflows/dependabot.yml --- .github/workflows/dependabot.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .github/workflows/dependabot.yml diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml deleted file mode 100644 index 12301490..00000000 --- a/.github/workflows/dependabot.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily"