From f46e165a81b23a102cc1855b29659240dffc2f15 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Mar 2019 16:05:54 -0800 Subject: [PATCH 01/13] Updated gitignore to exclude mypy cache --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 816aff376fc83..aa1d1a944a204 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ dist .coverage coverage.xml coverage_html_report +*.mypy_cache *.pytest_cache # hypothesis test database .hypothesis/ @@ -111,4 +112,4 @@ doc/build/html/index.html # Windows specific leftover: doc/tmp.sv env/ -doc/source/savefig/ +doc/source/savefig/ \ No newline at end of file From d358ab55e6f83335f28bc072458e9390cc42a97e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Mar 2019 16:06:09 -0800 Subject: [PATCH 02/13] Added initial mypy configuration files --- mypy.ini | 8 ++++++++ mypy_whitelist.txt | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 mypy.ini create mode 100644 mypy_whitelist.txt diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000000000..f7b2a747a91fb --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +follow_imports=silent + +[mypy-numpy.*] +ignore_missing_imports=True + +[mypy-pandas._libs.*] +ignore_missing_imports=True \ No newline at end of file diff --git a/mypy_whitelist.txt b/mypy_whitelist.txt new file mode 100644 index 0000000000000..578f9334998d0 --- /dev/null +++ b/mypy_whitelist.txt @@ -0,0 +1,17 @@ +pandas/core/dtypes/base.py +pandas/core/groupby/groupby.py +pandas/core/internals/blocks.py +pandas/core/internals/managers.py +pandas/core/common.py +pandas/core/arrays/datetimes.py +pandas/core/arrays/integer.py +pandas/core/arrays/period.py +pandas/core/arrays/sparse.py +pandas/core/arrays/datetimelike.py +pandas/core/arrays/array_.py +pandas/core/arrays/base.py +pandas/core/frame.py +pandas/core/base.py +pandas/core/indexes/period.py +pandas/core/indexes/datetimelike.py +pandas/core/indexes/base.py From db12d9fff04a11135ef6da3cd7295991420c6d26 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Mar 2019 16:11:06 -0800 Subject: [PATCH 03/13] dtypes/base.py cleanup --- pandas/core/dtypes/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/base.py b/pandas/core/dtypes/base.py index 88bbdcf342d66..8269f8c88ffd3 100644 --- a/pandas/core/dtypes/base.py +++ b/pandas/core/dtypes/base.py @@ -1,4 +1,6 @@ """Extend pandas with custom array types""" +from typing import List, Optional, Type + import numpy as np from pandas.errors import AbstractMethodError @@ -211,7 +213,7 @@ def __str__(self): @property def type(self): - # type: () -> type + # type: () -> Type """ The scalar type for the array, e.g. ``int`` From b77347a558d9812a947e1fa47ad447ef25ae7f62 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Mar 2019 16:18:34 -0800 Subject: [PATCH 04/13] Cleaned up groupby.groupby --- pandas/core/groupby/groupby.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 36dcb692bb079..8b91a7b244bf5 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -13,10 +13,11 @@ class providing the base-class of operations. from functools import partial, wraps import types import warnings +from typing import FrozenSet, Optional, Type import numpy as np -from pandas._libs import Timestamp, groupby as libgroupby +from pandas._libs import Timestamp, groupby as libgroupby # type: ignore import pandas.compat as compat from pandas.compat import range, set_function_name, zip from pandas.compat.numpy import function as nv @@ -325,7 +326,7 @@ def _group_selection_context(groupby): class _GroupBy(PandasObject, SelectionMixin): _group_selection = None - _apply_whitelist = frozenset() + _apply_whitelist = frozenset() # type: FrozenSet[str] def __init__(self, obj, keys=None, axis=0, level=None, grouper=None, exclusions=None, selection=None, as_index=True, @@ -1041,7 +1042,7 @@ def _bool_agg(self, val_test, skipna): """ def objs_to_bool(vals): - # type: (np.ndarray) -> (np.ndarray, typing.Type) + # type: (np.ndarray) -> (np.ndarray, Type) if is_object_dtype(vals): vals = np.array([bool(x) for x in vals]) else: @@ -1050,7 +1051,7 @@ def objs_to_bool(vals): return vals.view(np.uint8), np.bool def result_to_bool(result, inference): - # type: (np.ndarray, typing.Type) -> np.ndarray + # type: (np.ndarray, Type) -> np.ndarray return result.astype(inference, copy=False) return self._get_cythonized_result('group_any_all', self.grouper, @@ -1743,7 +1744,7 @@ def quantile(self, q=0.5, interpolation='linear'): """ def pre_processor(vals): - # type: (np.ndarray) -> (np.ndarray, Optional[typing.Type]) + # type: (np.ndarray) -> (np.ndarray, Optional[Type]) if is_object_dtype(vals): raise TypeError("'quantile' cannot be performed against " "'object' dtypes!") @@ -1758,7 +1759,7 @@ def pre_processor(vals): return vals, inference def post_processor(vals, inference): - # type: (np.ndarray, Optional[typing.Type]) -> np.ndarray + # type: (np.ndarray, Optional[Type]) -> np.ndarray if inference: # Check for edge case if not (is_integer_dtype(inference) and @@ -2021,7 +2022,7 @@ def _get_cythonized_result(self, how, grouper, aggregate=False, Function to be applied to result of Cython function. Should accept an array of values as the first argument and type inferences as its second argument, i.e. the signature should be - (ndarray, typing.Type). + (ndarray, Type). **kwargs : dict Extra arguments to be passed back to Cython funcs From 4c68a7b2e1c82e0c386482af28231d26ee4e2236 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Mar 2019 16:31:16 -0800 Subject: [PATCH 05/13] Fixed most of internals.blocks --- pandas/core/internals/blocks.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index ada663556899b..39b246eb63ddd 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -4,10 +4,12 @@ import inspect import re import warnings +from typing import Any, List, Optional import numpy as np -from pandas._libs import internals as libinternals, lib, tslib, tslibs +from pandas._libs import (internals as libinternals, # type: ignore + lib, tslib, tslibs) from pandas._libs.tslibs import Timedelta, conversion, is_null_datetimelike import pandas.compat as compat from pandas.compat import range, zip @@ -1826,8 +1828,12 @@ def interpolate(self, method='pad', axis=0, inplace=False, limit=None, limit=limit), placement=self.mgr_locs) - def shift(self, periods, axis=0, fill_value=None): - # type: (int, Optional[BlockPlacement], Any) -> List[ExtensionBlock] + def shift(self, + periods, # type: int + axis=0, # type: Optional[libinternals.BlockPlacement] + fill_value=None # type: Any + ): + # type: (...) -> List[ExtensionBlock] """ Shift the block by `periods`. From caec3239e83a1a8ec69d599924069e192f6bd4aa Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Mar 2019 16:35:31 -0800 Subject: [PATCH 06/13] internals.manager fixup --- pandas/core/internals/managers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 407db772d73e8..92339a76358b5 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -4,10 +4,12 @@ import itertools import operator import re +from typing import List, Optional, Union import numpy as np -from pandas._libs import internals as libinternals, lib +from pandas._libs import internals as libinternals, lib # type: ignore +from pandas.api.extensions import ExtensionDtype from pandas.compat import map, range, zip from pandas.util._validators import validate_bool_kwarg From fb987f4d94c4e4a4f6764a3c548477ab2e48f3a8 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Mar 2019 16:35:43 -0800 Subject: [PATCH 07/13] core.common fixup --- pandas/core/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/common.py b/pandas/core/common.py index 5b83cb344b1e7..77b7b94e7a1f7 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -9,6 +9,7 @@ from datetime import datetime, timedelta from functools import partial import inspect +from typing import Any import numpy as np From 9b5daaa41dadc8861b65a53321b928c498d4b9af Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Mar 2019 12:15:43 -0700 Subject: [PATCH 08/13] First cleanup pass for pandas.core.arrays --- pandas/core/arrays/array_.py | 7 ++++++- pandas/core/arrays/base.py | 8 +++++--- pandas/core/arrays/datetimelike.py | 6 ++++-- pandas/core/arrays/datetimes.py | 1 + pandas/core/arrays/integer.py | 2 +- pandas/core/arrays/period.py | 6 +++++- pandas/core/arrays/sparse.py | 7 ++++--- pandas/core/arrays/timedeltas.py | 5 +++-- 8 files changed, 29 insertions(+), 13 deletions(-) diff --git a/pandas/core/arrays/array_.py b/pandas/core/arrays/array_.py index 41d623c7efd9c..db3e9805b531d 100644 --- a/pandas/core/arrays/array_.py +++ b/pandas/core/arrays/array_.py @@ -1,8 +1,13 @@ +from typing import Optional, Sequence, Union + +import numpy as np + from pandas._libs import lib, tslibs +from pandas.core.arrays.base import ExtensionArray from pandas.core.dtypes.common import ( is_datetime64_ns_dtype, is_extension_array_dtype, is_timedelta64_ns_dtype) -from pandas.core.dtypes.dtypes import registry +from pandas.core.dtypes.dtypes import ExtensionDtype, registry from pandas import compat diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index e770281596134..f7d427ce26e6a 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -6,6 +6,7 @@ without warning. """ import operator +from typing import Any, Callable, Optional, Sequence, Tuple, Union import numpy as np @@ -15,6 +16,7 @@ from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.common import is_list_like +from pandas.core.dtypes.dtypes import ExtensionDtype from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries from pandas.core.dtypes.missing import isna @@ -365,7 +367,7 @@ def isna(self): raise AbstractMethodError(self) def _values_for_argsort(self): - # type: () -> ndarray + # type: () -> np.ndarray """ Return values for sorting. @@ -597,7 +599,7 @@ def searchsorted(self, value, side="left", sorter=None): return arr.searchsorted(value, side=side, sorter=sorter) def _values_for_factorize(self): - # type: () -> Tuple[ndarray, Any] + # type: () -> Tuple[np.ndarray, Any] """ Return an array and missing value suitable for factorization. @@ -622,7 +624,7 @@ def _values_for_factorize(self): return self.astype(object), np.nan def factorize(self, na_sentinel=-1): - # type: (int) -> Tuple[ndarray, ExtensionArray] + # type: (int) -> Tuple[np.ndarray, ExtensionArray] """ Encode the extension array as an enumerated type. diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 94668c74c1693..aded02d2dbde0 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -2,15 +2,17 @@ from datetime import datetime, timedelta import operator import warnings +from typing import Union, Sequence, Tuple import numpy as np from pandas._libs import NaT, algos, iNaT, lib +from pandas._libs.tslibs.nattype import NaTType from pandas._libs.tslibs.period import ( DIFFERENT_FREQ, IncompatibleFrequency, Period) from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds from pandas._libs.tslibs.timestamps import ( - RoundTo, maybe_integer_op_deprecated, round_nsint64) + RoundTo, maybe_integer_op_deprecated, round_nsint64, Timestamp) import pandas.compat as compat from pandas.compat.numpy import function as nv from pandas.errors import ( @@ -350,7 +352,7 @@ def __iter__(self): @property def asi8(self): - # type: () -> ndarray + # type: () -> np.ndarray """ Integer representation of the values. diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 75cf658423210..c2d6a0228e154 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -2,6 +2,7 @@ from datetime import datetime, time, timedelta import textwrap import warnings +from typing import Union import numpy as np from pytz import utc diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index fd90aec3b5e8c..0144d04c6e197 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -510,7 +510,7 @@ def value_counts(self, dropna=True): return Series(array, index=index) def _values_for_argsort(self): - # type: () -> ndarray + # type: () -> np.ndarray """Return values for sorting. Returns diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 0ec1bc7a84231..fff4734479792 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -1,17 +1,21 @@ # -*- coding: utf-8 -*- from datetime import timedelta import operator +from typing import Any, Callable, List, Optional, Sequence, Union import numpy as np from pandas._libs.tslibs import ( NaT, frequencies as libfrequencies, iNaT, period as libperiod) +from pandas._libs.tslibs.nattype import NaTType from pandas._libs.tslibs.fields import isleapyear_arr from pandas._libs.tslibs.period import ( DIFFERENT_FREQ, IncompatibleFrequency, Period, get_period_field_arr, period_asfreq_arr) from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds import pandas.compat as compat +from pandas.core.arrays.base import ExtensionArray +from pandas.core.indexes.base import Index from pandas.util._decorators import Appender, cache_readonly from pandas.core.dtypes.common import ( @@ -132,7 +136,7 @@ class PeriodArray(dtl.DatetimeLikeArrayMixin, dtl.DatelikeOps): _scalar_type = Period # Names others delegate to us - _other_ops = [] + _other_ops = [] # type: List[str] _bool_ops = ['is_leap_year'] _object_ops = ['start_time', 'end_time', 'freq'] _field_ops = ['year', 'month', 'day', 'hour', 'minute', 'second', diff --git a/pandas/core/arrays/sparse.py b/pandas/core/arrays/sparse.py index 9be2c9af169e8..3cce41cde4865 100644 --- a/pandas/core/arrays/sparse.py +++ b/pandas/core/arrays/sparse.py @@ -7,12 +7,13 @@ import operator import re import warnings +from typing import Any, Callable, Type, Union import numpy as np from pandas._libs import index as libindex, lib import pandas._libs.sparse as splib -from pandas._libs.sparse import BlockIndex, IntIndex +from pandas._libs.sparse import BlockIndex, IntIndex, SparseIndex from pandas._libs.tslibs import NaT import pandas.compat as compat from pandas.compat.numpy import function as nv @@ -79,7 +80,7 @@ class SparseDtype(ExtensionDtype): _metadata = ('_dtype', '_fill_value', '_is_na_fill_value') def __init__(self, dtype=np.float64, fill_value=None): - # type: (Union[str, np.dtype, 'ExtensionDtype', type], Any) -> None + # type: (Union[str, np.dtype, 'ExtensionDtype', Type], Any) -> None from pandas.core.dtypes.missing import na_value_for_dtype from pandas.core.dtypes.common import ( pandas_dtype, is_string_dtype, is_scalar @@ -372,7 +373,7 @@ def _subtype_with_str(self): def _get_fill(arr): - # type: (SparseArray) -> ndarray + # type: (SparseArray) -> np.ndarray """ Create a 0-dim ndarray containing the fill value diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 74fe8072e6924..d57fa0cfc2cbb 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -4,6 +4,7 @@ from datetime import timedelta import textwrap import warnings +from typing import List import numpy as np @@ -134,8 +135,8 @@ class TimedeltaArray(dtl.DatetimeLikeArrayMixin, dtl.TimelikeOps): _scalar_type = Timedelta __array_priority__ = 1000 # define my properties & methods for delegation - _other_ops = [] - _bool_ops = [] + _other_ops = [] # type: List[str] + _bool_ops = [] # type: List[str] _object_ops = ['freq'] _field_ops = ['days', 'seconds', 'microseconds', 'nanoseconds'] _datetimelike_ops = _field_ops + _object_ops + _bool_ops From d169cb6662f9c22488483179f6d4490b0f946201 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Mar 2019 12:35:37 -0700 Subject: [PATCH 09/13] pandas.core cleanups --- pandas/core/base.py | 1 + pandas/core/frame.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/base.py b/pandas/core/base.py index f896596dd5216..b357a47563f49 100644 --- a/pandas/core/base.py +++ b/pandas/core/base.py @@ -15,6 +15,7 @@ from pandas.util._decorators import Appender, Substitution, cache_readonly from pandas.util._validators import validate_bool_kwarg +from pandas.core.arrays.base import ExtensionArray from pandas.core.dtypes.common import ( is_datetime64_ns_dtype, is_datetime64tz_dtype, is_datetimelike, is_extension_array_dtype, is_extension_type, is_list_like, is_object_dtype, diff --git a/pandas/core/frame.py b/pandas/core/frame.py index eadffb779734f..ccf6d32a335cb 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -19,6 +19,7 @@ import sys import warnings from textwrap import dedent +from typing import List, Set, Union import numpy as np import numpy.ma as ma @@ -365,7 +366,7 @@ def _constructor(self): _constructor_sliced = Series _deprecations = NDFrame._deprecations | frozenset( ['get_value', 'set_value', 'from_csv', 'from_items']) - _accessors = set() + _accessors = set() # type: Set[str] @property def _constructor_expanddim(self): From 2abc6f553ba2100b00fb96d57302f540db81362c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Mar 2019 12:54:21 -0700 Subject: [PATCH 10/13] pandas.core.indexes cleanup --- pandas/core/indexes/base.py | 3 ++- pandas/core/indexes/datetimelike.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index dee181fc1c569..653b4d3966804 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -2,10 +2,11 @@ import operator from textwrap import dedent import warnings +from typing import Union import numpy as np -from pandas._libs import ( +from pandas._libs import ( # type: ignore algos as libalgos, index as libindex, join as libjoin, lib) from pandas._libs.lib import is_datetime_array from pandas._libs.tslibs import OutOfBoundsDatetime, Timedelta, Timestamp diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index aa7332472fc07..4e7d1b8180b1b 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -4,6 +4,7 @@ """ import operator import warnings +from typing import Set import numpy as np @@ -698,9 +699,9 @@ class DatetimelikeDelegateMixin(PandasDelegate): boxed in an index, after being returned from the array """ # raw_methods : dispatch methods that shouldn't be boxed in an Index - _raw_methods = set() + _raw_methods = set() # type: Set[str] # raw_properties : dispatch properties that shouldn't be boxed in an Index - _raw_properties = set() + _raw_properties = set() # type: Set[str] name = None _data = None From 2807e005167e605f815c3d06004c0950a8c3ea3b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Mar 2019 21:38:37 -0700 Subject: [PATCH 11/13] Alternate configuration --- mypy.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mypy.ini b/mypy.ini index f7b2a747a91fb..f8b37ee5b8663 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,8 +1,6 @@ [mypy] -follow_imports=silent - -[mypy-numpy.*] ignore_missing_imports=True +follow_imports=silent -[mypy-pandas._libs.*] -ignore_missing_imports=True \ No newline at end of file +[mypy-pandas.conftest,pandas.tests.*] +ignore_errors=True \ No newline at end of file From fb41b5ff40e9c30ad29ab7ab7cc72b1ad4e4c875 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 11 Mar 2019 12:38:15 -0700 Subject: [PATCH 12/13] Updated whitelist --- mypy_whitelist.txt | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/mypy_whitelist.txt b/mypy_whitelist.txt index 578f9334998d0..89af127af1be7 100644 --- a/mypy_whitelist.txt +++ b/mypy_whitelist.txt @@ -1,17 +1,9 @@ pandas/core/dtypes/base.py pandas/core/groupby/groupby.py -pandas/core/internals/blocks.py pandas/core/internals/managers.py pandas/core/common.py +pandas/core/arrays/timedeltas.py pandas/core/arrays/datetimes.py -pandas/core/arrays/integer.py -pandas/core/arrays/period.py -pandas/core/arrays/sparse.py -pandas/core/arrays/datetimelike.py -pandas/core/arrays/array_.py pandas/core/arrays/base.py pandas/core/frame.py -pandas/core/base.py -pandas/core/indexes/period.py -pandas/core/indexes/datetimelike.py pandas/core/indexes/base.py From ac7e76895b80314c78b246abb19a8764516123ae Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 11 Mar 2019 12:39:57 -0700 Subject: [PATCH 13/13] Reverted newline deletion --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aa1d1a944a204..91e9d9be54b81 100644 --- a/.gitignore +++ b/.gitignore @@ -112,4 +112,4 @@ doc/build/html/index.html # Windows specific leftover: doc/tmp.sv env/ -doc/source/savefig/ \ No newline at end of file +doc/source/savefig/